From dc94eade03022ce3a5286f5e781576321a5f1653 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Fri, 27 Apr 2012 08:59:00 -0500
Subject: Incomplete-only todolists optimization

We can push this down to the database if we know in advance we only need
the incomplete lists. This helps our call on the developer dashboard
quite a bit; the time of the single query in question drops from >1300ms
to around 40ms.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/views.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

(limited to 'devel')

diff --git a/devel/views.py b/devel/views.py
index d2ce65db..39f28a65 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -49,8 +49,7 @@ def index(request):
     todopkgs = todopkgs.filter(pkg__pkgbase__in=inner_q).order_by(
             'list__name', 'pkg__pkgname')
 
-    todolists = get_annotated_todolists()
-    todolists = [todolist for todolist in todolists if todolist.incomplete_count > 0]
+    todolists = get_annotated_todolists(incomplete_only=True)
 
     signoffs = sorted(get_signoff_groups(user=request.user),
             key=operator.attrgetter('pkgbase'))
-- 
cgit v1.2.3-2-g168b


From 80e7d19726a95b40727b7f35b9ad80b436b14b93 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Fri, 27 Apr 2012 09:24:34 -0500
Subject: Dev dashboard performance improvement

Rather than one query per cell in the arches and repos statistics
tables, we can group these together up front using Django annotations.
This means we only need one query per table.

In my local instance with all of the staging repos imported, this
reduces the total query count on this page from 56 to 26, a rather
marked improvement.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/views.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

(limited to 'devel')

diff --git a/devel/views.py b/devel/views.py
index 39f28a65..cf0d8ad2 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -13,7 +13,7 @@ from django.contrib.auth.models import User, Group
 from django.contrib.sites.models import Site
 from django.core.mail import send_mail
 from django.db import transaction
-from django.db.models import F
+from django.db.models import F, Count
 from django.http import Http404
 from django.shortcuts import get_object_or_404
 from django.template import loader, Context
@@ -54,6 +54,11 @@ def index(request):
     signoffs = sorted(get_signoff_groups(user=request.user),
             key=operator.attrgetter('pkgbase'))
 
+    arches = Arch.objects.all().annotate(
+            total_ct=Count('packages'), flagged_ct=Count('packages__flag_date'))
+    repos = Repo.objects.all().annotate(
+            total_ct=Count('packages'), flagged_ct=Count('packages__flag_date'))
+
     maintainers = get_annotated_maintainers()
 
     maintained = PackageRelation.objects.filter(
@@ -70,12 +75,12 @@ def index(request):
 
     page_dict = {
             'todos': todolists,
-            'repos': Repo.objects.all(),
-            'arches': Arch.objects.all(),
+            'arches': arches,
+            'repos': repos,
             'maintainers': maintainers,
             'orphan': orphan,
-            'flagged' : flagged,
-            'todopkgs' : todopkgs,
+            'flagged': flagged,
+            'todopkgs': todopkgs,
             'signoffs': signoffs
     }
 
-- 
cgit v1.2.3-2-g168b


From 25a2fbc7c1cb50fa80ed4de50830721507765a91 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Sun, 29 Apr 2012 19:15:19 -0500
Subject: Add a "last action" column to developer clocks page

This allows people to easily see if a developer has done anything
recently that we can easily grab a date for. Obviously this doesn't
include all sources of activity, so the list of things checked is
clearly stated at the top.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/views.py | 43 ++++++++++++++++++++++++++++++++++++-------
 1 file changed, 36 insertions(+), 7 deletions(-)

(limited to 'devel')

diff --git a/devel/views.py b/devel/views.py
index cf0d8ad2..85acda74 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -1,4 +1,4 @@
-from datetime import date, datetime, timedelta
+from datetime import timedelta
 import operator
 import pytz
 import random
@@ -9,24 +9,28 @@ from django import forms
 from django.http import HttpResponseRedirect
 from django.contrib.auth.decorators import \
         login_required, permission_required, user_passes_test
+from django.contrib.admin.models import LogEntry, ADDITION
 from django.contrib.auth.models import User, Group
+from django.contrib.contenttypes.models import ContentType
 from django.contrib.sites.models import Site
 from django.core.mail import send_mail
 from django.db import transaction
-from django.db.models import F, Count
+from django.db.models import F, Count, Max
 from django.http import Http404
 from django.shortcuts import get_object_or_404
 from django.template import loader, Context
 from django.template.defaultfilters import filesizeformat
 from django.views.decorators.cache import never_cache
 from django.views.generic.simple import direct_to_template
+from django.utils.encoding import force_unicode
 from django.utils.http import http_date
 
 from .models import UserProfile
 from main.models import Package, PackageDepend, PackageFile, TodolistPkg
 from main.models import Arch, Repo
 from main.utils import utc_now
-from packages.models import PackageRelation
+from news.models import News
+from packages.models import PackageRelation, Signoff
 from packages.utils import get_signoff_groups
 from todolists.utils import get_annotated_todolists
 from .utils import get_annotated_maintainers, UserFinder
@@ -91,6 +95,33 @@ def clock(request):
     devs = User.objects.filter(is_active=True).order_by(
             'first_name', 'last_name').select_related('userprofile')
 
+    latest_news = dict(News.objects.filter(
+            author__is_active=True).values_list('author').order_by(
+            ).annotate(last_post=Max('postdate')))
+    latest_package = dict(Package.objects.filter(
+            packager__is_active=True).values_list('packager').order_by(
+            ).annotate(last_build=Max('build_date')))
+    latest_signoff = dict(Signoff.objects.filter(
+            user__is_active=True).values_list('user').order_by(
+            ).annotate(last_signoff=Max('created')))
+    latest_log = dict(LogEntry.objects.filter(
+            user__is_active=True).values_list('user').order_by(
+            ).annotate(last_log=Max('action_time')))
+
+    for dev in devs:
+        dates = [
+            latest_news.get(dev.id, None),
+            latest_package.get(dev.id, None),
+            latest_signoff.get(dev.id, None),
+            latest_log.get(dev.id, None),
+            dev.last_login,
+        ]
+        dates = [d for d in dates if d is not None]
+        if dates:
+            dev.last_action = max(dates)
+        else:
+            dev.last_action = None
+
     now = utc_now()
     page_dict = {
             'developers': devs,
@@ -135,7 +166,8 @@ class UserProfileForm(forms.ModelForm):
 def change_profile(request):
     if request.POST:
         form = ProfileForm(request.POST)
-        profile_form = UserProfileForm(request.POST, request.FILES, instance=request.user.get_profile())
+        profile_form = UserProfileForm(request.POST, request.FILES,
+                instance=request.user.get_profile())
         if form.is_valid() and profile_form.is_valid():
             request.user.email = form.cleaned_data['email']
             if form.cleaned_data['passwd1']:
@@ -331,9 +363,6 @@ class NewUserForm(forms.ModelForm):
 
 def log_addition(request, obj):
     """Cribbed from ModelAdmin.log_addition."""
-    from django.contrib.admin.models import LogEntry, ADDITION
-    from django.contrib.contenttypes.models import ContentType
-    from django.utils.encoding import force_unicode
     LogEntry.objects.log_action(
         user_id         = request.user.pk,
         content_type_id = ContentType.objects.get_for_model(obj).pk,
-- 
cgit v1.2.3-2-g168b


From badc535aeb1d310a9b8aa59aade07045e6eae653 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Wed, 18 Apr 2012 15:05:43 -0500
Subject: Ensure order_by default value is cleared when using distinct()

Otherwise the queryset returns nonsensical results. I find the design of
this less than obvious but so be it; we can ensure the results work
regardless of a default ordering on the model.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/views.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'devel')

diff --git a/devel/views.py b/devel/views.py
index 85acda74..7ef33362 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -249,7 +249,8 @@ def report(request, report_name, username=None):
         if username:
             pkg_ids = set(packages.values_list('id', flat=True))
             bad_files = bad_files.filter(pkg__in=pkg_ids)
-        bad_files = bad_files.values_list('pkg_id', flat=True).distinct()
+        bad_files = bad_files.values_list(
+                'pkg_id', flat=True).order_by().distinct()
         packages = packages.filter(id__in=set(bad_files))
     elif report_name == 'uncompressed-info':
         title = 'Packages with uncompressed infopages'
@@ -260,7 +261,8 @@ def report(request, report_name, username=None):
         if username:
             pkg_ids = set(packages.values_list('id', flat=True))
             bad_files = bad_files.filter(pkg__in=pkg_ids)
-        bad_files = bad_files.values_list('pkg_id', flat=True).distinct()
+        bad_files = bad_files.values_list(
+                'pkg_id', flat=True).order_by().distinct()
         packages = packages.filter(id__in=set(bad_files))
     elif report_name == 'unneeded-orphans':
         title = 'Orphan packages required by no other packages'
-- 
cgit v1.2.3-2-g168b


From cf8ecdf9fce0573ad207d024708f21a5dbbbb120 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Wed, 2 May 2012 09:46:52 -0500
Subject: rematch_developers: do mass updates instead of single saves

When updating a lot of objects, it makes much more sense to perform
targeted update queries rather than one-row-at-a-time saves.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/rematch_developers.py | 61 ++++++++++++-------------
 1 file changed, 30 insertions(+), 31 deletions(-)

(limited to 'devel')

diff --git a/devel/management/commands/rematch_developers.py b/devel/management/commands/rematch_developers.py
index 8383cc8d..ab2f0f4b 100644
--- a/devel/management/commands/rematch_developers.py
+++ b/devel/management/commands/rematch_developers.py
@@ -46,52 +46,51 @@ class Command(NoArgsCommand):
 
 @transaction.commit_on_success
 def match_packager(finder):
-    logger.info("getting all unmatched packages")
+    logger.info("getting all unmatched packager strings")
     package_count = matched_count = 0
-    unknown = set()
-
-    for package in Package.objects.filter(packager__isnull=True):
-        if package.packager_str in unknown:
-            continue
-        logger.debug("package %s, packager string %s",
-                package.pkgname, package.packager_str)
-        package_count += 1
-        user = finder.find(package.packager_str)
+    mapping = {}
+
+    unmatched = Package.objects.filter(packager__isnull=True).values_list(
+            'packager_str', flat=True).order_by().distinct()
+
+    for packager in unmatched:
+        logger.debug("packager string %s", packager)
+        user = finder.find(packager)
         if user:
-            package.packager = user
+            mapping[packager] = user
             logger.debug("  found user %s" % user.username)
-            package.save()
             matched_count += 1
-        else:
-            unknown.add(package.packager_str)
 
-    logger.info("%d packager strings checked, %d newly matched",
+    for packager_str, user in mapping.items():
+        package_count += Package.objects.filter(packager__isnull=True,
+                packager_str=packager_str).update(packager=user)
+
+    logger.info("%d packages updated, %d packager strings matched",
             package_count, matched_count)
-    logger.debug("unknown packagers:\n%s",
-            "\n".join(unknown))
 
 
 @transaction.commit_on_success
 def match_flagrequest(finder):
-    logger.info("getting all non-user flag requests")
+    logger.info("getting all flag requests emails from unknown users")
     req_count = matched_count = 0
-    unknown = set()
-
-    for request in FlagRequest.objects.filter(user__isnull=True):
-        if request.user_email in unknown:
-            continue
-        logger.debug("email %s", request.user_email)
-        req_count += 1
-        user = finder.find_by_email(request.user_email)
+    mapping = {}
+
+    unmatched = FlagRequest.objects.filter(user__isnull=True).values_list(
+            'user_email', flat=True).order_by().distinct()
+    
+    for user_email in unmatched:
+        logger.debug("email %s", user_email)
+        user = finder.find_by_email(user_email)
         if user:
-            request.user = user
+            mapping[user_email] = user
             logger.debug("  found user %s" % user.username)
-            request.save()
             matched_count += 1
-        else:
-            unknown.add(request.user_email)
 
-    logger.info("%d request emails checked, %d newly matched",
+    for user_email, user in mapping.items():
+        req_count += FlagRequest.objects.filter(user__isnull=True,
+                user_email=user_email).update(user=user)
+
+    logger.info("%d request emails updated, %d emails matched",
             req_count, matched_count)
 
 # vim: set ts=4 sw=4 et:
-- 
cgit v1.2.3-2-g168b


From f36d876aca5571f09032d0d2a67c8b1f1a3258c8 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Sat, 12 May 2012 09:22:52 -0500
Subject: Change to new time access methods in pgpdump code

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'devel')

diff --git a/devel/views.py b/devel/views.py
index 7ef33362..0f1c8d15 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -280,7 +280,7 @@ def report(request, report_name, username=None):
         filtered = []
         packages = packages.filter(pgp_signature__isnull=False)
         for package in packages:
-            sig_date = package.signature.datetime.replace(tzinfo=pytz.utc)
+            sig_date = package.signature.creation_time.replace(tzinfo=pytz.utc)
             package.sig_date = sig_date.date()
             key_id = package.signature.key_id
             signer = finder.find_by_pgp_key(key_id)
-- 
cgit v1.2.3-2-g168b


From 72a92102df4999dbcc370064707c9026d51c4fe7 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Fri, 18 May 2012 21:29:03 -0500
Subject: Switch to usage of new Depend object

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py | 18 ++++++++++--------
 devel/views.py                        |  6 +++---
 2 files changed, 13 insertions(+), 11 deletions(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index fd8e3979..47294d9a 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -29,9 +29,9 @@ from django.db import connections, router, transaction
 from django.db.utils import IntegrityError
 
 from devel.utils import UserFinder
-from main.models import Arch, Package, PackageDepend, PackageFile, Repo
+from main.models import Arch, Package, PackageFile, Repo
 from main.utils import utc_now
-from packages.models import Conflict, Provision, Replacement
+from packages.models import Depend, Conflict, Provision, Replacement
 
 
 logging.basicConfig(
@@ -141,19 +141,21 @@ class RepoPackage(object):
         return u'%s-%s' % (self.ver, self.rel)
 
 
-DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.*))?$")
+DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.+))?$")
 
 def create_depend(package, dep_str, optional=False):
-    depend = PackageDepend(pkg=package, optional=optional)
+    depend = Depend(pkg=package, optional=optional)
     # lop off any description first
     parts = dep_str.split(':', 1)
     if len(parts) > 1:
         depend.description = parts[1].strip()
     match = DEPEND_RE.match(parts[0].strip())
     if match:
-        depend.depname = match.group(1)
-        if match.group(2):
-            depend.depvcmp = match.group(2)
+        depend.name = match.group(1)
+        if match.group(3):
+            depend.comparison = match.group(3)
+        if match.group(4):
+            related.version = match.group(4)
     else:
         logger.warning('Package %s had unparsable depend string %s',
                 package.pkgname, dep_str)
@@ -232,7 +234,7 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None):
     dbpkg.depends.all().delete()
     deps = [create_depend(dbpkg, y) for y in repopkg.depends]
     deps += [create_depend(dbpkg, y, True) for y in repopkg.optdepends]
-    PackageDepend.objects.bulk_create(deps)
+    Depend.objects.bulk_create(deps)
 
     dbpkg.conflicts.all().delete()
     conflicts = [create_related(Conflict, dbpkg, y) for y in repopkg.conflicts]
diff --git a/devel/views.py b/devel/views.py
index 0f1c8d15..16b6acc6 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -26,11 +26,11 @@ from django.utils.encoding import force_unicode
 from django.utils.http import http_date
 
 from .models import UserProfile
-from main.models import Package, PackageDepend, PackageFile, TodolistPkg
+from main.models import Package, PackageFile, TodolistPkg
 from main.models import Arch, Repo
 from main.utils import utc_now
 from news.models import News
-from packages.models import PackageRelation, Signoff
+from packages.models import PackageRelation, Signoff, Depend
 from packages.utils import get_signoff_groups
 from todolists.utils import get_annotated_todolists
 from .utils import get_annotated_maintainers, UserFinder
@@ -267,7 +267,7 @@ def report(request, report_name, username=None):
     elif report_name == 'unneeded-orphans':
         title = 'Orphan packages required by no other packages'
         owned = PackageRelation.objects.all().values('pkgbase')
-        required = PackageDepend.objects.all().values('depname')
+        required = Depend.objects.all().values('name')
         # The two separate calls to exclude is required to do the right thing
         packages = packages.exclude(pkgbase__in=owned).exclude(
                 pkgname__in=required)
-- 
cgit v1.2.3-2-g168b


From 3f9aeb45c2a2b498a389bacc92b5f56d9feb4329 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Sat, 19 May 2012 09:54:15 -0500
Subject: reporead: fix copy/paste issue

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index 47294d9a..2e8c4625 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -155,7 +155,7 @@ def create_depend(package, dep_str, optional=False):
         if match.group(3):
             depend.comparison = match.group(3)
         if match.group(4):
-            related.version = match.group(4)
+            depend.version = match.group(4)
     else:
         logger.warning('Package %s had unparsable depend string %s',
                 package.pkgname, dep_str)
-- 
cgit v1.2.3-2-g168b


From 1b03069f0d9e0397a7ff07404343c9400bbcfa1c Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Thu, 7 Jun 2012 20:53:52 -0500
Subject: Use 3 decimal places for showing compression ratio

Otherwise there are too many grouped under each value.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'devel')

diff --git a/devel/views.py b/devel/views.py
index 16b6acc6..78ed26f2 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -234,7 +234,7 @@ def report(request, report_name, username=None):
             package.installed_size_pretty = filesizeformat(
                 package.installed_size)
             ratio = package.compressed_size / float(package.installed_size)
-            package.ratio = '%.2f' % ratio
+            package.ratio = '%.3f' % ratio
             package.compress_type = package.filename.split('.')[-1]
     elif report_name == 'uncompressed-man':
         title = 'Packages with uncompressed manpages'
-- 
cgit v1.2.3-2-g168b


From bbcbde0197d4862b5acc595b17bc5051780dbc9e Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Wed, 20 Jun 2012 17:03:26 -0500
Subject: Add a last_modified field to user profiles

A behind the scenes field that might be slightly useful.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 ...08_auto__add_field_userprofile_last_modified.py | 108 +++++++++++++++++++++
 devel/models.py                                    |  18 +++-
 2 files changed, 125 insertions(+), 1 deletion(-)
 create mode 100644 devel/migrations/0008_auto__add_field_userprofile_last_modified.py

(limited to 'devel')

diff --git a/devel/migrations/0008_auto__add_field_userprofile_last_modified.py b/devel/migrations/0008_auto__add_field_userprofile_last_modified.py
new file mode 100644
index 00000000..2695987a
--- /dev/null
+++ b/devel/migrations/0008_auto__add_field_userprofile_last_modified.py
@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+    def forwards(self, orm):
+        db.add_column('user_profiles', 'last_modified',
+                      self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime(2000, 1, 1, 0, 0)),
+                      keep_default=False)
+
+    def backwards(self, orm):
+        db.delete_column('user_profiles', 'last_modified')
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'devel.masterkey': {
+            'Meta': {'ordering': "('created',)", 'object_name': 'MasterKey'},
+            'created': ('django.db.models.fields.DateField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'masterkey_owner'", 'to': "orm['auth.User']"}),
+            'pgp_key': ('devel.fields.PGPKeyField', [], {'max_length': '40'}),
+            'revoked': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'revoker': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'masterkey_revoker'", 'to': "orm['auth.User']"})
+        },
+        'devel.pgpsignature': {
+            'Meta': {'object_name': 'PGPSignature'},
+            'created': ('django.db.models.fields.DateField', [], {}),
+            'expires': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'signee': ('devel.fields.PGPKeyField', [], {'max_length': '40'}),
+            'signer': ('devel.fields.PGPKeyField', [], {'max_length': '40'}),
+            'valid': ('django.db.models.fields.BooleanField', [], {'default': 'True'})
+        },
+        'devel.userprofile': {
+            'Meta': {'object_name': 'UserProfile', 'db_table': "'user_profiles'"},
+            'alias': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'allowed_repos': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['main.Repo']", 'symmetrical': 'False', 'blank': 'True'}),
+            'country': ('django_countries.fields.CountryField', [], {'max_length': '2', 'blank': 'True'}),
+            'favorite_distros': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'interests': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'languages': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'last_modified': ('django.db.models.fields.DateTimeField', [], {}),
+            'latin_name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'location': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'notify': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'occupation': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'other_contact': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'pgp_key': ('devel.fields.PGPKeyField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}),
+            'picture': ('django.db.models.fields.files.FileField', [], {'default': "'devs/silhouette.png'", 'max_length': '100'}),
+            'public_email': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'roles': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'time_zone': ('django.db.models.fields.CharField', [], {'default': "'UTC'", 'max_length': '100'}),
+            'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'userprofile'", 'unique': 'True', 'to': "orm['auth.User']"}),
+            'website': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'yob': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'})
+        },
+        'main.repo': {
+            'Meta': {'ordering': "['name']", 'object_name': 'Repo', 'db_table': "'repos'"},
+            'bugs_category': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}),
+            'bugs_project': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+            'staging': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'svn_root': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'testing': ('django.db.models.fields.BooleanField', [], {'default': 'False'})
+        }
+    }
+
+    complete_apps = ['devel']
diff --git a/devel/models.py b/devel/models.py
index fd5a0347..fd5df00a 100644
--- a/devel/models.py
+++ b/devel/models.py
@@ -2,11 +2,12 @@
 import pytz
 
 from django.db import models
+from django.db.models.signals import pre_save
 from django.contrib.auth.models import User
 from django_countries import CountryField
 
 from .fields import PGPKeyField
-from main.utils import make_choice
+from main.utils import make_choice, utc_now
 
 
 class UserProfile(models.Model):
@@ -44,6 +45,7 @@ class UserProfile(models.Model):
     allowed_repos = models.ManyToManyField('main.Repo', blank=True)
     latin_name = models.CharField(max_length=255, null=True, blank=True,
         help_text="Latin-form name; used only for non-Latin full names")
+    last_modified = models.DateTimeField(editable=False)
 
     class Meta:
         db_table = 'user_profiles'
@@ -96,4 +98,18 @@ class PGPSignature(models.Model):
     def __unicode__(self):
         return u'%s → %s' % (self.signer, self.signee)
 
+
+def set_last_modified(sender, **kwargs):
+    '''This will set the 'last_modified' field on the user profile to the
+    current UTC time when either the profile is updated.  For use as a pre_save
+    signal handler.'''
+    obj = kwargs['instance']
+    if hasattr(obj, 'last_modified'):
+        obj.last_modified = utc_now()
+
+
+# connect signals needed to keep cache in line with reality
+pre_save.connect(set_last_modified, sender=UserProfile,
+        dispatch_uid="devel.models")
+
 # vim: set ts=4 sw=4 et:
-- 
cgit v1.2.3-2-g168b


From a87fe016d1a1bf7fdcd2b19f515aa72a5b93db2b Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Sun, 1 Jul 2012 20:21:34 -0500
Subject: Log package updates during reporead invocation

This adds a Manager and log_update method to help log all updates made
to the packages table during reporead runs.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index 2e8c4625..4e242af1 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -14,6 +14,7 @@ Example:
 """
 
 from collections import defaultdict
+from copy import copy
 import io
 import os
 import re
@@ -31,7 +32,7 @@ from django.db.utils import IntegrityError
 from devel.utils import UserFinder
 from main.models import Arch, Package, PackageFile, Repo
 from main.utils import utc_now
-from packages.models import Depend, Conflict, Provision, Replacement
+from packages.models import Depend, Conflict, Provision, Replacement, Update
 
 
 logging.basicConfig(
@@ -362,6 +363,7 @@ def db_update(archname, reponame, pkgs, force=False):
         try:
             with transaction.commit_on_success():
                 populate_pkg(dbpkg, pkg, timestamp=utc_now())
+                Update.objects.log_update(None, dbpkg)
         except IntegrityError:
             logger.warning("Could not add package %s; "
                     "not fatal if another thread beat us to it.",
@@ -372,6 +374,7 @@ def db_update(archname, reponame, pkgs, force=False):
         logger.info("Removing package %s", pkgname)
         dbpkg = dbdict[pkgname]
         with transaction.commit_on_success():
+            Update.objects.log_update(dbpkg, None)
             # no race condition here as long as simultaneous threads both
             # issue deletes; second delete will be a no-op
             delete_pkg_files(dbpkg)
@@ -399,7 +402,9 @@ def db_update(archname, reponame, pkgs, force=False):
                 logger.debug("Package %s was already updated", pkg.name)
                 continue
             logger.info("Updating package %s", pkg.name)
+            prevpkg = copy(dbpkg)
             populate_pkg(dbpkg, pkg, force=force, timestamp=timestamp)
+            Update.objects.log_update(prevpkg, dbpkg)
 
     logger.info('Finished updating arch: %s', archname)
 
-- 
cgit v1.2.3-2-g168b


From daf011b67a338f26ead8058a9f9caedfe251c62c Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Thu, 5 Jul 2012 11:25:00 -0400
Subject: reporead: properly handle cases where last_update ==
 files_last_update

We should assume the filelists are up to date in this case, not out of
date.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index 4e242af1..51c73c02 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -273,7 +273,7 @@ def populate_files(dbpkg, repopkg, force=False):
             return
         if not dbpkg.files_last_update or not dbpkg.last_update:
             pass
-        elif dbpkg.files_last_update > dbpkg.last_update:
+        elif dbpkg.files_last_update >= dbpkg.last_update:
             return
 
     # only delete files if we are reading a DB that contains them
@@ -427,7 +427,7 @@ def filesonly_update(archname, reponame, pkgs, force=False):
         with transaction.commit_on_success():
             if not dbpkg.files_last_update or not dbpkg.last_update:
                 pass
-            elif not force and dbpkg.files_last_update > dbpkg.last_update:
+            elif not force and dbpkg.files_last_update >= dbpkg.last_update:
                 logger.debug("Files for %s are up to date", pkg.name)
                 continue
             dbpkg = Package.objects.select_for_update().get(id=dbpkg.id)
-- 
cgit v1.2.3-2-g168b


From 9d91cad678133e97345111fab2c103fcda9b9f28 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Thu, 5 Jul 2012 11:25:40 -0400
Subject: reporead: handle files in root directory properly

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index 51c73c02..df29a8a7 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -283,7 +283,10 @@ def populate_files(dbpkg, repopkg, force=False):
                 len(repopkg.files), dbpkg.pkgname)
         pkg_files = []
         for f in repopkg.files:
-            dirname, filename = f.rsplit('/', 1)
+            if '/' in f:
+                dirname, filename = f.rsplit('/', 1)
+            else:
+                dirname, filename = '', f
             if filename == '':
                 filename = None
             pkgfile = PackageFile(pkg=dbpkg,
-- 
cgit v1.2.3-2-g168b


From 909cb9a209b4a4db00232b3a62656f95c4b88d45 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Thu, 5 Jul 2012 17:09:55 -0500
Subject: reporead: don't append slash to empty (root) directory

Add the slash only if we have a directory name, and not otherwise.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index df29a8a7..43578d4a 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -285,13 +285,14 @@ def populate_files(dbpkg, repopkg, force=False):
         for f in repopkg.files:
             if '/' in f:
                 dirname, filename = f.rsplit('/', 1)
+                dirname += '/'
             else:
                 dirname, filename = '', f
             if filename == '':
                 filename = None
             pkgfile = PackageFile(pkg=dbpkg,
                     is_directory=(filename is None),
-                    directory=dirname + '/',
+                    directory=dirname,
                     filename=filename)
             pkg_files.append(pkgfile)
         PackageFile.objects.bulk_create(pkg_files)
-- 
cgit v1.2.3-2-g168b


From a1ec14fc68282d67c00c79b5aa6aab60461f056a Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Mon, 12 Mar 2012 13:14:49 -0400
Subject: reporead: disable FULL synchronous writes for sqlite3

At least on Linux, we hit a huge bottleneck waiting for the FULL commit
to happen for each added package during reporead operations. It makes
much more sense to back this off to FULL level instead, which trades
some possible loss of durability for speedier operation. Additionally,
no one would possibly be running their production version of this site
on sqlite3, right?

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index 43578d4a..e50686b1 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -547,6 +547,12 @@ def read_repo(primary_arch, repo_file, options):
                     package.name, repo_file, package.arch))
     del packages
 
+    database = router.db_for_write(Package)
+    connection = connections[database]
+    if connection.vendor == 'sqlite':
+        cursor = connection.cursor()
+        cursor.execute('PRAGMA synchronous = NORMAL')
+
     logger.info('Starting database updates for %s.', repo_file)
     for arch in sorted(packages_arches.keys()):
         if filesonly:
-- 
cgit v1.2.3-2-g168b


From 88ee61a39ac3690267f2b7903f3646972e8f055d Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Sun, 8 Jul 2012 21:24:48 -0500
Subject: Work around bulk_create limitations in sqlite3 in reporead

Given the 999 SQL statement variable limit, we can easily hit it when
updating a package with thousands of files or a few hundred depends.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py | 34 ++++++++++++++++++++++++++++------
 1 file changed, 28 insertions(+), 6 deletions(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index e50686b1..2d9b68b2 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -31,7 +31,7 @@ from django.db.utils import IntegrityError
 
 from devel.utils import UserFinder
 from main.models import Arch, Package, PackageFile, Repo
-from main.utils import utc_now
+from main.utils import utc_now, database_vendor
 from packages.models import Depend, Conflict, Provision, Replacement, Update
 
 
@@ -184,6 +184,28 @@ def create_related(model, package, rel_str, equals_only=False):
         return None
     return related
 
+
+def batched_bulk_create(model, all_objects):
+    # for short lists, just bulk_create as we should be fine
+    if len(all_objects) < 20:
+        return model.objects.bulk_create(all_objects)
+
+    if database_vendor(model, mode='write') == 'sqlite':
+        # 999 max variables in each SQL statement
+        incr = 999 // len(model._meta.fields)
+    else:
+        incr = 1000
+
+    def chunks():
+        offset = 0
+        while offset < len(all_objects):
+            yield all_objects[offset:offset + incr]
+            offset += incr
+
+    for items in chunks():
+        model.objects.bulk_create(items)
+
+
 def create_multivalued(dbpkg, repopkg, db_attr, repo_attr):
     '''Populate the simplest of multivalued attributes. These are those that
     only deal with a 'name' attribute, such as licenses, groups, etc. The input
@@ -235,20 +257,20 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None):
     dbpkg.depends.all().delete()
     deps = [create_depend(dbpkg, y) for y in repopkg.depends]
     deps += [create_depend(dbpkg, y, True) for y in repopkg.optdepends]
-    Depend.objects.bulk_create(deps)
+    batched_bulk_create(Depend, deps)
 
     dbpkg.conflicts.all().delete()
     conflicts = [create_related(Conflict, dbpkg, y) for y in repopkg.conflicts]
-    Conflict.objects.bulk_create(conflicts)
+    batched_bulk_create(Conflict, conflicts)
 
     dbpkg.provides.all().delete()
     provides = [create_related(Provision, dbpkg, y, equals_only=True)
             for y in repopkg.provides]
-    Provision.objects.bulk_create(provides)
+    batched_bulk_create(Provision, provides)
 
     dbpkg.replaces.all().delete()
     replaces = [create_related(Replacement, dbpkg, y) for y in repopkg.replaces]
-    Replacement.objects.bulk_create(replaces)
+    batched_bulk_create(Replacement, replaces)
 
     create_multivalued(dbpkg, repopkg, 'groups', 'groups')
     create_multivalued(dbpkg, repopkg, 'licenses', 'license')
@@ -295,7 +317,7 @@ def populate_files(dbpkg, repopkg, force=False):
                     directory=dirname,
                     filename=filename)
             pkg_files.append(pkgfile)
-        PackageFile.objects.bulk_create(pkg_files)
+        batched_bulk_create(PackageFile, pkg_files)
         dbpkg.files_last_update = utc_now()
         dbpkg.save()
 
-- 
cgit v1.2.3-2-g168b


From c0bf9e20660cfae7ea8994472555bba23398b598 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Tue, 24 Jul 2012 09:19:48 -0500
Subject: Remove custom utc_now() function, use django.utils.timezone.now()

This was around from the time when we handled timezones sanely and
Django did not; now that we are on 1.4 we no longer need our own code to
handle this.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py |  9 +++++----
 devel/models.py                       |  5 +++--
 devel/views.py                        | 12 ++++++------
 3 files changed, 14 insertions(+), 12 deletions(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index 2d9b68b2..e69691db 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -28,10 +28,11 @@ from pytz import utc
 from django.core.management.base import BaseCommand, CommandError
 from django.db import connections, router, transaction
 from django.db.utils import IntegrityError
+from django.utils.timezone import now
 
 from devel.utils import UserFinder
 from main.models import Arch, Package, PackageFile, Repo
-from main.utils import utc_now, database_vendor
+from main.utils import database_vendor
 from packages.models import Depend, Conflict, Provision, Replacement, Update
 
 
@@ -318,7 +319,7 @@ def populate_files(dbpkg, repopkg, force=False):
                     filename=filename)
             pkg_files.append(pkgfile)
         batched_bulk_create(PackageFile, pkg_files)
-        dbpkg.files_last_update = utc_now()
+        dbpkg.files_last_update = now()
         dbpkg.save()
 
 
@@ -388,7 +389,7 @@ def db_update(archname, reponame, pkgs, force=False):
         dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository)
         try:
             with transaction.commit_on_success():
-                populate_pkg(dbpkg, pkg, timestamp=utc_now())
+                populate_pkg(dbpkg, pkg, timestamp=now())
                 Update.objects.log_update(None, dbpkg)
         except IntegrityError:
             logger.warning("Could not add package %s; "
@@ -417,7 +418,7 @@ def db_update(archname, reponame, pkgs, force=False):
         if not force and pkg_same_version(pkg, dbpkg):
             continue
         elif not force:
-            timestamp = utc_now()
+            timestamp = now()
 
         # The odd select_for_update song and dance here are to ensure
         # simultaneous updates don't happen on a package, causing
diff --git a/devel/models.py b/devel/models.py
index fd5df00a..9b6f07a7 100644
--- a/devel/models.py
+++ b/devel/models.py
@@ -4,10 +4,11 @@ import pytz
 from django.db import models
 from django.db.models.signals import pre_save
 from django.contrib.auth.models import User
+from django.utils.timezone import now
 from django_countries import CountryField
 
 from .fields import PGPKeyField
-from main.utils import make_choice, utc_now
+from main.utils import make_choice
 
 
 class UserProfile(models.Model):
@@ -105,7 +106,7 @@ def set_last_modified(sender, **kwargs):
     signal handler.'''
     obj = kwargs['instance']
     if hasattr(obj, 'last_modified'):
-        obj.last_modified = utc_now()
+        obj.last_modified = now()
 
 
 # connect signals needed to keep cache in line with reality
diff --git a/devel/views.py b/devel/views.py
index 78ed26f2..143b12bf 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -24,11 +24,11 @@ from django.views.decorators.cache import never_cache
 from django.views.generic.simple import direct_to_template
 from django.utils.encoding import force_unicode
 from django.utils.http import http_date
+from django.utils.timezone import now
 
 from .models import UserProfile
 from main.models import Package, PackageFile, TodolistPkg
 from main.models import Arch, Repo
-from main.utils import utc_now
 from news.models import News
 from packages.models import PackageRelation, Signoff, Depend
 from packages.utils import get_signoff_groups
@@ -122,15 +122,15 @@ def clock(request):
         else:
             dev.last_action = None
 
-    now = utc_now()
+    current_time = now()
     page_dict = {
             'developers': devs,
-            'utc_now': now,
+            'utc_now': current_time,
     }
 
     response = direct_to_template(request, 'devel/clock.html', page_dict)
     if not response.has_header('Expires'):
-        expire_time = now.replace(second=0, microsecond=0)
+        expire_time = current_time.replace(second=0, microsecond=0)
         expire_time += timedelta(minutes=1)
         expire_time = time.mktime(expire_time.timetuple())
         response['Expires'] = http_date(expire_time)
@@ -198,12 +198,12 @@ def report(request, report_name, username=None):
 
     if report_name == 'old':
         title = 'Packages last built more than one year ago'
-        cutoff = utc_now() - timedelta(days=365)
+        cutoff = now() - timedelta(days=365)
         packages = packages.filter(
                 build_date__lt=cutoff).order_by('build_date')
     elif report_name == 'long-out-of-date':
         title = 'Packages marked out-of-date more than 90 days ago'
-        cutoff = utc_now() - timedelta(days=90)
+        cutoff = now() - timedelta(days=90)
         packages = packages.filter(
                 flag_date__lt=cutoff).order_by('flag_date')
     elif report_name == 'big':
-- 
cgit v1.2.3-2-g168b


From 76c37ce3acc7a4af0271c7535d4a33042f7749b5 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Tue, 24 Jul 2012 09:35:55 -0500
Subject: Replace deprecated direct_to_template() with render() shortcut

Now that Django actually provides a concise way to use a RequestContext
object without instantiating it, we can use that rather than the old
function-based generic view that worked well to do the same.
Additionally, these function-based generic views will be gone in Django
1.5, so might as well make the move now.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/views.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

(limited to 'devel')

diff --git a/devel/views.py b/devel/views.py
index 143b12bf..f877bc84 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -17,11 +17,10 @@ from django.core.mail import send_mail
 from django.db import transaction
 from django.db.models import F, Count, Max
 from django.http import Http404
-from django.shortcuts import get_object_or_404
+from django.shortcuts import get_object_or_404, render
 from django.template import loader, Context
 from django.template.defaultfilters import filesizeformat
 from django.views.decorators.cache import never_cache
-from django.views.generic.simple import direct_to_template
 from django.utils.encoding import force_unicode
 from django.utils.http import http_date
 from django.utils.timezone import now
@@ -88,7 +87,7 @@ def index(request):
             'signoffs': signoffs
     }
 
-    return direct_to_template(request, 'devel/index.html', page_dict)
+    return render(request, 'devel/index.html', page_dict)
 
 @login_required
 def clock(request):
@@ -128,7 +127,7 @@ def clock(request):
             'utc_now': current_time,
     }
 
-    response = direct_to_template(request, 'devel/clock.html', page_dict)
+    response = render(request, 'devel/clock.html', page_dict)
     if not response.has_header('Expires'):
         expire_time = current_time.replace(second=0, microsecond=0)
         expire_time += timedelta(minutes=1)
@@ -178,7 +177,7 @@ def change_profile(request):
     else:
         form = ProfileForm(initial={'email': request.user.email})
         profile_form = UserProfileForm(instance=request.user.get_profile())
-    return direct_to_template(request, 'devel/profile.html',
+    return render(request, 'devel/profile.html',
             {'form': form, 'profile_form': profile_form})
 
 @login_required
@@ -301,7 +300,7 @@ def report(request, report_name, username=None):
         'column_names': names,
         'column_attrs': attrs,
     }
-    return direct_to_template(request, 'devel/packages.html', context)
+    return render(request, 'devel/packages.html', context)
 
 
 class NewUserForm(forms.ModelForm):
@@ -399,7 +398,7 @@ def new_user_form(request):
         'title': 'Create User',
         'submit_text': 'Create User'
     }
-    return direct_to_template(request, 'general_form.html', context)
+    return render(request, 'general_form.html', context)
 
 @user_passes_test(lambda u: u.is_superuser)
 def admin_log(request, username=None):
@@ -410,6 +409,6 @@ def admin_log(request, username=None):
         'title': "Admin Action Log",
         'log_user':  user,
     }
-    return direct_to_template(request, 'devel/admin_log.html', context)
+    return render(request, 'devel/admin_log.html', context)
 
 # vim: set ts=4 sw=4 et:
-- 
cgit v1.2.3-2-g168b


From 280c53eec5661252b5692fa374292c4d421e3bd8 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Sat, 28 Jul 2012 11:45:15 -0500
Subject: reporead: don't use iexact lookup on arch name

We don't do this anywhere else, so we shouldn't do this here either.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index e69691db..aaa9812e 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -332,7 +332,7 @@ def update_common(archname, reponame, pkgs, sanity_check=True):
         transaction.set_dirty()
 
         repository = Repo.objects.get(name__iexact=reponame)
-        architecture = Arch.objects.get(name__iexact=archname)
+        architecture = Arch.objects.get(name=archname)
         # no-arg order_by() removes even the default ordering; we don't need it
         dbpkgs = Package.objects.filter(
                 arch=architecture, repo=repository).order_by()
@@ -371,7 +371,7 @@ def db_update(archname, reponame, pkgs, force=False):
     logger.info('Updating %s (%s)', reponame, archname)
     dbpkgs = update_common(archname, reponame, pkgs, True)
     repository = Repo.objects.get(name__iexact=reponame)
-    architecture = Arch.objects.get(name__iexact=archname)
+    architecture = Arch.objects.get(name=archname)
 
     # This makes our inner loop where we find packages by name *way* more
     # efficient by not having to go to the database for each package to
@@ -538,7 +538,7 @@ def locate_arch(arch):
     if isinstance(arch, Arch):
         return arch
     try:
-        return Arch.objects.get(name__iexact=arch)
+        return Arch.objects.get(name=arch)
     except Arch.DoesNotExist:
         raise CommandError(
                 'Specified architecture %s is not currently known.' % arch)
-- 
cgit v1.2.3-2-g168b


From 3f0c024754047d92e8ce4aa4ecf93a06865f8448 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Tue, 31 Jul 2012 18:37:30 -0500
Subject: PGP key handling updates

* Import signatures for all known keys, not just active developers
* Ensure we are only showing and accounting for active developers on the
  master keys page
* Add a new table showing signatures between developers

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/generate_keyring.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'devel')

diff --git a/devel/management/commands/generate_keyring.py b/devel/management/commands/generate_keyring.py
index b9117c84..15ae488d 100644
--- a/devel/management/commands/generate_keyring.py
+++ b/devel/management/commands/generate_keyring.py
@@ -48,7 +48,7 @@ def generate_keyring(keyserver, keyring):
     logger.info("getting all known key IDs")
 
     # Screw you Django, for not letting one natively do value != <empty string>
-    key_ids = UserProfile.objects.filter(user__is_active=True,
+    key_ids = UserProfile.objects.filter(
             pgp_key__isnull=False).extra(where=["pgp_key != ''"]).values_list(
             "pgp_key", flat=True)
     logger.info("%d keys fetched from user profiles", len(key_ids))
-- 
cgit v1.2.3-2-g168b


From a64bbbd4139d91cbbca10d804067cbd87a95872d Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Tue, 31 Jul 2012 20:27:43 -0500
Subject: Make adjustments for optional -> deptype conversion

Very little dealt directly with this field.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index aaa9812e..a3bf3e0c 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -145,8 +145,8 @@ class RepoPackage(object):
 
 DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.+))?$")
 
-def create_depend(package, dep_str, optional=False):
-    depend = Depend(pkg=package, optional=optional)
+def create_depend(package, dep_str, deptype='D'):
+    depend = Depend(pkg=package, deptype=deptype)
     # lop off any description first
     parts = dep_str.split(':', 1)
     if len(parts) > 1:
@@ -257,7 +257,7 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None):
 
     dbpkg.depends.all().delete()
     deps = [create_depend(dbpkg, y) for y in repopkg.depends]
-    deps += [create_depend(dbpkg, y, True) for y in repopkg.optdepends]
+    deps += [create_depend(dbpkg, y, 'O') for y in repopkg.optdepends]
     batched_bulk_create(Depend, deps)
 
     dbpkg.conflicts.all().delete()
-- 
cgit v1.2.3-2-g168b


From 1f2466fffceafebfaca34e3ed2d34de6b622768b Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Tue, 31 Jul 2012 20:35:50 -0500
Subject: reporead: import make and check depends

We don't have these in the database yet, but future verisons of repo-add
will put this information in the sync databases.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index a3bf3e0c..8b55b09a 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -80,8 +80,9 @@ class RepoPackage(object):
     bare = ( 'name', 'base', 'arch', 'filename',
             'md5sum', 'sha256sum', 'url', 'packager' )
     number = ( 'csize', 'isize' )
-    collections = ( 'depends', 'optdepends', 'conflicts',
-            'provides', 'replaces', 'groups', 'license', 'files' )
+    collections = ( 'depends', 'optdepends', 'makedepends', 'checkdepends',
+            'conflicts', 'provides', 'replaces', 'groups', 'license',
+            'files' )
 
     version_re = re.compile(r'^((\d+):)?(.+)-([^-]+)$')
 
@@ -258,6 +259,8 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None):
     dbpkg.depends.all().delete()
     deps = [create_depend(dbpkg, y) for y in repopkg.depends]
     deps += [create_depend(dbpkg, y, 'O') for y in repopkg.optdepends]
+    deps += [create_depend(dbpkg, y, 'M') for y in repopkg.makedepends]
+    deps += [create_depend(dbpkg, y, 'C') for y in repopkg.checkdepends]
     batched_bulk_create(Depend, deps)
 
     dbpkg.conflicts.all().delete()
-- 
cgit v1.2.3-2-g168b


From 978a5c61a5412eeed054307d3e2979324ffcb64a Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Wed, 8 Aug 2012 19:34:37 -0500
Subject: Add flag requests to developer last action calculation

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/views.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

(limited to 'devel')

diff --git a/devel/views.py b/devel/views.py
index f877bc84..ad1f4deb 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -29,7 +29,7 @@ from .models import UserProfile
 from main.models import Package, PackageFile, TodolistPkg
 from main.models import Arch, Repo
 from news.models import News
-from packages.models import PackageRelation, Signoff, Depend
+from packages.models import PackageRelation, Signoff, FlagRequest, Depend
 from packages.utils import get_signoff_groups
 from todolists.utils import get_annotated_todolists
 from .utils import get_annotated_maintainers, UserFinder
@@ -103,6 +103,11 @@ def clock(request):
     latest_signoff = dict(Signoff.objects.filter(
             user__is_active=True).values_list('user').order_by(
             ).annotate(last_signoff=Max('created')))
+    # The extra() bit ensures we can use our 'user_id IS NOT NULL' index
+    latest_flagreq = dict(FlagRequest.objects.filter(
+            user__is_active=True).extra(
+            where=['user_id IS NOT NULL']).values_list('user_id').order_by(
+            ).annotate(last_flagrequest=Max('created')))
     latest_log = dict(LogEntry.objects.filter(
             user__is_active=True).values_list('user').order_by(
             ).annotate(last_log=Max('action_time')))
@@ -112,6 +117,7 @@ def clock(request):
             latest_news.get(dev.id, None),
             latest_package.get(dev.id, None),
             latest_signoff.get(dev.id, None),
+            latest_flagreq.get(dev.id, None),
             latest_log.get(dev.id, None),
             dev.last_login,
         ]
-- 
cgit v1.2.3-2-g168b


From 241ff8fbd79f9f17cd326a34eb39096851f630ba Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Wed, 8 Aug 2012 22:07:06 -0500
Subject: Extract parse_version function from reporead logic

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index 8b55b09a..af0a2dc0 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -34,6 +34,7 @@ from devel.utils import UserFinder
 from main.models import Arch, Package, PackageFile, Repo
 from main.utils import database_vendor
 from packages.models import Depend, Conflict, Provision, Replacement, Update
+from packages.utils import parse_version
 
 
 logging.basicConfig(
@@ -84,8 +85,6 @@ class RepoPackage(object):
             'conflicts', 'provides', 'replaces', 'groups', 'license',
             'files' )
 
-    version_re = re.compile(r'^((\d+):)?(.+)-([^-]+)$')
-
     def __init__(self, repo):
         self.repo = repo
         self.ver = None
@@ -112,11 +111,7 @@ class RepoPackage(object):
                 # do NOT prune these values at all
                 setattr(self, k, v[0])
             elif k == 'version':
-                match = self.version_re.match(v[0])
-                self.ver = match.group(3)
-                self.rel = match.group(4)
-                if match.group(2):
-                    self.epoch = int(match.group(2))
+                self.ver, self.rel, self.epoch = parse_version(v[0])
             elif k == 'builddate':
                 try:
                     builddate = datetime.utcfromtimestamp(int(v[0]))
-- 
cgit v1.2.3-2-g168b


From ca0011c585ec28f9dde0f400a77fd6f859d520b0 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Wed, 15 Aug 2012 08:11:00 -0500
Subject: Add ability to rematch developers on <username>@archlinux.org
 addresses

This makes this matcher catch a bit more with the wide net we were
already casting.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/rematch_developers.py |  4 +-
 devel/utils.py                                  | 50 +++++++++++++++++--------
 2 files changed, 38 insertions(+), 16 deletions(-)

(limited to 'devel')

diff --git a/devel/management/commands/rematch_developers.py b/devel/management/commands/rematch_developers.py
index ab2f0f4b..2b379588 100644
--- a/devel/management/commands/rematch_developers.py
+++ b/devel/management/commands/rematch_developers.py
@@ -53,6 +53,7 @@ def match_packager(finder):
     unmatched = Package.objects.filter(packager__isnull=True).values_list(
             'packager_str', flat=True).order_by().distinct()
 
+    logger.info("%d packager strings retrieved", len(unmatched))
     for packager in unmatched:
         logger.debug("packager string %s", packager)
         user = finder.find(packager)
@@ -71,13 +72,14 @@ def match_packager(finder):
 
 @transaction.commit_on_success
 def match_flagrequest(finder):
-    logger.info("getting all flag requests emails from unknown users")
+    logger.info("getting all flag request email addresses from unknown users")
     req_count = matched_count = 0
     mapping = {}
 
     unmatched = FlagRequest.objects.filter(user__isnull=True).values_list(
             'user_email', flat=True).order_by().distinct()
     
+    logger.info("%d email addresses retrieved", len(unmatched))
     for user_email in unmatched:
         logger.debug("email %s", user_email)
         user = finder.find_by_email(user_email)
diff --git a/devel/utils.py b/devel/utils.py
index 85b4e42f..e8e3a6c4 100644
--- a/devel/utils.py
+++ b/devel/utils.py
@@ -1,6 +1,7 @@
 import re
 
 from django.contrib.auth.models import User
+from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
 from django.db import connection
 from django.db.models import Count, Q
 
@@ -44,6 +45,15 @@ SELECT pr.user_id, COUNT(*), COUNT(p.flag_date)
     return maintainers
 
 
+def ignore_does_not_exist(func):
+    def new_func(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except (ObjectDoesNotExist, MultipleObjectsReturned):
+            return None
+    return new_func
+
+
 class UserFinder(object):
     def __init__(self):
         self.cache = {}
@@ -52,18 +62,33 @@ class UserFinder(object):
         self.pgp_cache = {}
 
     @staticmethod
+    @ignore_does_not_exist
     def user_email(name, email):
         if email:
             return User.objects.get(email=email)
         return None
 
     @staticmethod
+    @ignore_does_not_exist
+    def username_email(name, email):
+        if email and '@' in email:
+            # split email addr at '@' symbol, ensure domain matches
+            # or is a subdomain of archlinux.org
+            # TODO: configurable domain/regex somewhere?
+            username, domain = email.split('@', 1)
+            if re.match(r'^(.+\.)?archlinux.org$', domain):
+                return User.objects.get(username=username)
+        return None
+
+    @staticmethod
+    @ignore_does_not_exist
     def profile_email(name, email):
         if email:
             return User.objects.get(userprofile__public_email=email)
         return None
 
     @staticmethod
+    @ignore_does_not_exist
     def user_name(name, email):
         # yes, a bit odd but this is the easiest way since we can't always be
         # sure how to split the name. Ensure every 'token' appears in at least
@@ -102,14 +127,12 @@ class UserFinder(object):
             email = matches.group(2)
 
         user = None
-        find_methods = (self.user_email, self.profile_email, self.user_name)
+        find_methods = (self.user_email, self.profile_email,
+                self.username_email, self.user_name)
         for matcher in find_methods:
-            try:
-                user = matcher(name, email)
-                if user != None:
-                    break
-            except (User.DoesNotExist, User.MultipleObjectsReturned):
-                pass
+            user = matcher(name, email)
+            if user != None:
+                break
 
         self.cache[userstring] = user
         self.email_cache[email] = user
@@ -135,14 +158,11 @@ class UserFinder(object):
         if email in self.email_cache:
             return self.email_cache[email]
 
-        user = None
-        try:
-            user = self.user_email(None, email)
-        except User.DoesNotExist:
-            try:
-                user = self.profile_email(None, email)
-            except User.DoesNotExist:
-                pass
+        user = self.user_email(None, email)
+        if user is None:
+            user = self.profile_email(None, email)
+            if user is None:
+                user = self.username_email(None, email)
 
         self.email_cache[email] = user
         return user
-- 
cgit v1.2.3-2-g168b


From e7e9b151643772f2bf9564d215ec8b90cd9b45c6 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Wed, 5 Sep 2012 08:44:33 -0500
Subject: Split devel forms out of devel/views.py

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/forms.py |  99 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 devel/views.py | 106 ++++++---------------------------------------------------
 2 files changed, 109 insertions(+), 96 deletions(-)
 create mode 100644 devel/forms.py

(limited to 'devel')

diff --git a/devel/forms.py b/devel/forms.py
new file mode 100644
index 00000000..861a576c
--- /dev/null
+++ b/devel/forms.py
@@ -0,0 +1,99 @@
+import random
+from string import ascii_letters, digits
+
+from django import forms
+from django.contrib.auth.models import User, Group
+from django.contrib.sites.models import Site
+from django.core.mail import send_mail
+from django.template import loader, Context
+
+from .models import UserProfile
+
+
+class ProfileForm(forms.Form):
+    email = forms.EmailField(label='Private email (not shown publicly):',
+            help_text="Used for out-of-date notifications, etc.")
+    passwd1 = forms.CharField(label='New Password', required=False,
+            widget=forms.PasswordInput)
+    passwd2 = forms.CharField(label='Confirm Password', required=False,
+            widget=forms.PasswordInput)
+
+    def clean(self):
+        if self.cleaned_data['passwd1'] != self.cleaned_data['passwd2']:
+            raise forms.ValidationError('Passwords do not match.')
+        return self.cleaned_data
+
+
+class UserProfileForm(forms.ModelForm):
+    def clean_pgp_key(self):
+        data = self.cleaned_data['pgp_key']
+        # strip 0x prefix if provided; store uppercase
+        if data.startswith('0x'):
+            data = data[2:]
+        return data.upper()
+
+    class Meta:
+        model = UserProfile
+        exclude = ('allowed_repos', 'user', 'latin_name')
+
+
+class NewUserForm(forms.ModelForm):
+    username = forms.CharField(max_length=30)
+    private_email = forms.EmailField()
+    first_name = forms.CharField(required=False)
+    last_name = forms.CharField(required=False)
+    groups = forms.ModelMultipleChoiceField(required=False,
+            queryset=Group.objects.all())
+
+    class Meta:
+        model = UserProfile
+        exclude = ('picture', 'user')
+
+    def __init__(self, *args, **kwargs):
+        super(NewUserForm, self).__init__(*args, **kwargs)
+        # Hack ourself so certain fields appear first. self.fields is a
+        # SortedDict object where we can manipulate the keyOrder list.
+        order = self.fields.keyOrder
+        keys = ('username', 'private_email', 'first_name', 'last_name')
+        for key in reversed(keys):
+            order.remove(key)
+            order.insert(0, key)
+
+    def clean_username(self):
+        username = self.cleaned_data['username']
+        if User.objects.filter(username=username).exists():
+            raise forms.ValidationError(
+                    "A user with that username already exists.")
+        return username
+
+    def save(self, commit=True):
+        profile = super(NewUserForm, self).save(False)
+        pwletters = ascii_letters + digits
+        password = ''.join([random.choice(pwletters) for _ in xrange(8)])
+        user = User.objects.create_user(username=self.cleaned_data['username'],
+                email=self.cleaned_data['private_email'], password=password)
+        user.first_name = self.cleaned_data['first_name']
+        user.last_name = self.cleaned_data['last_name']
+        user.save()
+        # sucks that the MRM.add() method can't take a list directly... we have
+        # to resort to dirty * magic.
+        user.groups.add(*self.cleaned_data['groups'])
+        profile.user = user
+        if commit:
+            profile.save()
+            self.save_m2m()
+
+        template = loader.get_template('devel/new_account.txt')
+        ctx = Context({
+            'site': Site.objects.get_current(),
+            'user': user,
+            'password': password,
+        })
+
+        send_mail("Your new archweb account",
+                template.render(ctx),
+                'Arch Website Notification <nobody@archlinux.org>',
+                [user.email],
+                fail_silently=False)
+
+# vim: set ts=4 sw=4 et:
diff --git a/devel/views.py b/devel/views.py
index ad1f4deb..5406974e 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -1,31 +1,25 @@
 from datetime import timedelta
 import operator
 import pytz
-import random
-from string import ascii_letters, digits
 import time
 
-from django import forms
 from django.http import HttpResponseRedirect
 from django.contrib.auth.decorators import \
         login_required, permission_required, user_passes_test
 from django.contrib.admin.models import LogEntry, ADDITION
-from django.contrib.auth.models import User, Group
+from django.contrib.auth.models import User
 from django.contrib.contenttypes.models import ContentType
-from django.contrib.sites.models import Site
-from django.core.mail import send_mail
 from django.db import transaction
 from django.db.models import F, Count, Max
 from django.http import Http404
 from django.shortcuts import get_object_or_404, render
-from django.template import loader, Context
 from django.template.defaultfilters import filesizeformat
 from django.views.decorators.cache import never_cache
 from django.utils.encoding import force_unicode
 from django.utils.http import http_date
 from django.utils.timezone import now
 
-from .models import UserProfile
+from .forms import ProfileForm, UserProfileForm, NewUserForm
 from main.models import Package, PackageFile, TodolistPkg
 from main.models import Arch, Repo
 from news.models import News
@@ -89,6 +83,7 @@ def index(request):
 
     return render(request, 'devel/index.html', page_dict)
 
+
 @login_required
 def clock(request):
     devs = User.objects.filter(is_active=True).order_by(
@@ -141,30 +136,6 @@ def clock(request):
         response['Expires'] = http_date(expire_time)
     return response
 
-class ProfileForm(forms.Form):
-    email = forms.EmailField(label='Private email (not shown publicly):',
-            help_text="Used for out-of-date notifications, etc.")
-    passwd1 = forms.CharField(label='New Password', required=False,
-            widget=forms.PasswordInput)
-    passwd2 = forms.CharField(label='Confirm Password', required=False,
-            widget=forms.PasswordInput)
-
-    def clean(self):
-        if self.cleaned_data['passwd1'] != self.cleaned_data['passwd2']:
-            raise forms.ValidationError('Passwords do not match.')
-        return self.cleaned_data
-
-class UserProfileForm(forms.ModelForm):
-    def clean_pgp_key(self):
-        data = self.cleaned_data['pgp_key']
-        # strip 0x prefix if provided; store uppercase
-        if data.startswith('0x'):
-            data = data[2:]
-        return data.upper()
-
-    class Meta:
-        model = UserProfile
-        exclude = ('allowed_repos', 'user', 'latin_name')
 
 @login_required
 @never_cache
@@ -177,8 +148,9 @@ def change_profile(request):
             request.user.email = form.cleaned_data['email']
             if form.cleaned_data['passwd1']:
                 request.user.set_password(form.cleaned_data['passwd1'])
-            request.user.save()
-            profile_form.save()
+            with transaction.commit_on_success():
+                request.user.save()
+                profile_form.save()
             return HttpResponseRedirect('/devel/')
     else:
         form = ProfileForm(initial={'email': request.user.email})
@@ -186,6 +158,7 @@ def change_profile(request):
     return render(request, 'devel/profile.html',
             {'form': form, 'profile_form': profile_form})
 
+
 @login_required
 def report(request, report_name, username=None):
     title = 'Developer Report'
@@ -309,65 +282,6 @@ def report(request, report_name, username=None):
     return render(request, 'devel/packages.html', context)
 
 
-class NewUserForm(forms.ModelForm):
-    username = forms.CharField(max_length=30)
-    private_email = forms.EmailField()
-    first_name = forms.CharField(required=False)
-    last_name = forms.CharField(required=False)
-    groups = forms.ModelMultipleChoiceField(required=False,
-            queryset=Group.objects.all())
-
-    class Meta:
-        model = UserProfile
-        exclude = ('picture', 'user')
-
-    def __init__(self, *args, **kwargs):
-        super(NewUserForm, self).__init__(*args, **kwargs)
-        # Hack ourself so certain fields appear first. self.fields is a
-        # SortedDict object where we can manipulate the keyOrder list.
-        order = self.fields.keyOrder
-        keys = ('username', 'private_email', 'first_name', 'last_name')
-        for key in reversed(keys):
-            order.remove(key)
-            order.insert(0, key)
-
-    def clean_username(self):
-        username = self.cleaned_data['username']
-        if User.objects.filter(username=username).exists():
-            raise forms.ValidationError(
-                    "A user with that username already exists.")
-        return username
-
-    def save(self, commit=True):
-        profile = super(NewUserForm, self).save(False)
-        pwletters = ascii_letters + digits
-        password = ''.join([random.choice(pwletters) for _ in xrange(8)])
-        user = User.objects.create_user(username=self.cleaned_data['username'],
-                email=self.cleaned_data['private_email'], password=password)
-        user.first_name = self.cleaned_data['first_name']
-        user.last_name = self.cleaned_data['last_name']
-        user.save()
-        # sucks that the MRM.add() method can't take a list directly... we have
-        # to resort to dirty * magic.
-        user.groups.add(*self.cleaned_data['groups'])
-        profile.user = user
-        if commit:
-            profile.save()
-            self.save_m2m()
-
-        template = loader.get_template('devel/new_account.txt')
-        ctx = Context({
-            'site': Site.objects.get_current(),
-            'user': user,
-            'password': password,
-        })
-
-        send_mail("Your new archweb account",
-                template.render(ctx),
-                'Arch Website Notification <nobody@archlinux.org>',
-                [user.email],
-                fail_silently=False)
-
 def log_addition(request, obj):
     """Cribbed from ModelAdmin.log_addition."""
     LogEntry.objects.log_action(
@@ -379,17 +293,16 @@ def log_addition(request, obj):
         change_message  = "Added via Create New User form."
     )
 
+
 @permission_required('auth.add_user')
 @never_cache
 def new_user_form(request):
     if request.POST:
         form = NewUserForm(request.POST)
         if form.is_valid():
-            @transaction.commit_on_success
-            def inner_save():
+            with transaction.commit_on_success():
                 form.save()
                 log_addition(request, form.instance.user)
-            inner_save()
             return HttpResponseRedirect('/admin/auth/user/%d/' % \
                     form.instance.user.id)
     else:
@@ -406,6 +319,7 @@ def new_user_form(request):
     }
     return render(request, 'general_form.html', context)
 
+
 @user_passes_test(lambda u: u.is_superuser)
 def admin_log(request, username=None):
     user = None
-- 
cgit v1.2.3-2-g168b


From a2034fc80d4e73816502537f8dfe864ab4ef8db3 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Sat, 15 Sep 2012 09:14:36 -0500
Subject: Add JS-based filtering to the developer reports

This can use the todolist filtering functions we made more generic in a
previous commit.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/views.py | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'devel')

diff --git a/devel/views.py b/devel/views.py
index 5406974e..23ff9f74 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -271,11 +271,15 @@ def report(request, report_name, username=None):
     else:
         raise Http404
 
+    arches = set(pkg.arch for pkg in packages)
+    repos = set(pkg.repo for pkg in packages)
     context = {
         'all_maintainers': maints,
         'title': title,
         'maintainer': user,
         'packages': packages,
+        'arches': sorted(arches),
+        'repos': sorted(repos),
         'column_names': names,
         'column_attrs': attrs,
     }
-- 
cgit v1.2.3-2-g168b


From c76e5c768687394b8022883d01edf85dc3c30e7f Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Mon, 17 Sep 2012 21:42:48 -0500
Subject: Sort package list before inserting it into the database

FS#30323. This will take some time to propagate to all existing
packages, but all new and updated packages will start getting filelists
in the right order.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index af0a2dc0..ac745092 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -303,7 +303,10 @@ def populate_files(dbpkg, repopkg, force=False):
         logger.info("adding %d files for package %s",
                 len(repopkg.files), dbpkg.pkgname)
         pkg_files = []
-        for f in repopkg.files:
+        # sort in normal alpha-order that pacman uses, rather than makepkg's
+        # default breadth-first, directory-first ordering
+        files = sorted(repopkg.files)
+        for f in files:
             if '/' in f:
                 dirname, filename = f.rsplit('/', 1)
                 dirname += '/'
-- 
cgit v1.2.3-2-g168b


From 7d5cfe45d52c4dbd2f431f0edcafc9936b740ab2 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Thu, 20 Sep 2012 09:32:42 -0500
Subject: Explicitly close the database connection in reporead

This is the cause of these warnings showing up in the PostgreSQL log:
    LOG:  unexpected EOF on client connection with an open transaction

All management commands are guilty of this as they do not clean up and
close the connection when they exit, unlike the standard web request
cycle. Other commands should probably be updated as well, but for now,
this is the biggest culprit.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py | 1 +
 1 file changed, 1 insertion(+)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index ac745092..ce5c8cb7 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -584,6 +584,7 @@ def read_repo(primary_arch, repo_file, options):
         else:
             db_update(arch, repo, packages_arches[arch], force)
     logger.info('Finished database updates for %s.', repo_file)
+    connection.close()
     return 0
 
 # vim: set ts=4 sw=4 et:
-- 
cgit v1.2.3-2-g168b


From 89c6ffc95cb1d5fe4bd2534562ca732d727a8686 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Thu, 20 Sep 2012 09:34:46 -0500
Subject: chmod -x reporead_inotify.py

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead_inotify.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100755 => 100644 devel/management/commands/reporead_inotify.py

(limited to 'devel')

diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py
old mode 100755
new mode 100644
-- 
cgit v1.2.3-2-g168b


From 05f309d7e57a66d9309abbf19b4328bad514b978 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Mon, 24 Sep 2012 21:13:02 -0500
Subject: Add a new column to developer repo stats

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/views.py | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'devel')

diff --git a/devel/views.py b/devel/views.py
index 23ff9f74..ea85a901 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -55,6 +55,11 @@ def index(request):
             total_ct=Count('packages'), flagged_ct=Count('packages__flag_date'))
     repos = Repo.objects.all().annotate(
             total_ct=Count('packages'), flagged_ct=Count('packages__flag_date'))
+    # the join is huge unless we do this separately, so merge the result here
+    repo_maintainers = dict(Repo.objects.all().values_list('id').annotate(
+            Count('userprofile')))
+    for repo in repos:
+        repo.maintainer_ct = repo_maintainers.get(repo.id, 0)
 
     maintainers = get_annotated_maintainers()
 
-- 
cgit v1.2.3-2-g168b


From a1c4d831c92cbb32cb8c34f95b8cd5eb541cdf00 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Mon, 24 Sep 2012 21:23:18 -0500
Subject: Exclude inactive developers in maintainer count

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/views.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'devel')

diff --git a/devel/views.py b/devel/views.py
index ea85a901..083665d9 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -56,7 +56,8 @@ def index(request):
     repos = Repo.objects.all().annotate(
             total_ct=Count('packages'), flagged_ct=Count('packages__flag_date'))
     # the join is huge unless we do this separately, so merge the result here
-    repo_maintainers = dict(Repo.objects.all().values_list('id').annotate(
+    repo_maintainers = dict(Repo.objects.order_by().filter(
+            userprofile__user__is_active=True).values_list('id').annotate(
             Count('userprofile')))
     for repo in repos:
         repo.maintainer_ct = repo_maintainers.get(repo.id, 0)
-- 
cgit v1.2.3-2-g168b


From 3530303c9a7d017bdfec40d9dc7c38bd3fb2c09b Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Tue, 25 Sep 2012 18:21:10 -0500
Subject: Only watch non-staging repos in inotify reporead

This is temporary until we do more work to ensure staging packages don't
show up and confuse regular users of the web interface.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead_inotify.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py
index c74762eb..043e13fe 100644
--- a/devel/management/commands/reporead_inotify.py
+++ b/devel/management/commands/reporead_inotify.py
@@ -68,7 +68,7 @@ class Command(BaseCommand):
         and passes these on to the various pyinotify pieces as necessary and
         finally builds and returns a notifier object.'''
         arches = Arch.objects.filter(agnostic=False)
-        repos = Repo.objects.all()
+        repos = Repo.objects.filter(staging=False)
         arch_path_map = dict((arch, None) for arch in arches)
         all_paths = set()
         total_paths = 0
-- 
cgit v1.2.3-2-g168b


From ed1adeb1254c4d5754260bbe1ae2fbbc2a88debb Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Sun, 30 Sep 2012 02:02:04 -0500
Subject: Begin importing staging repos

This reverts 3530303c9a7d now that we have reasonably hidden most
staging package confusion on the site for normal end users.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead_inotify.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py
index 043e13fe..c74762eb 100644
--- a/devel/management/commands/reporead_inotify.py
+++ b/devel/management/commands/reporead_inotify.py
@@ -68,7 +68,7 @@ class Command(BaseCommand):
         and passes these on to the various pyinotify pieces as necessary and
         finally builds and returns a notifier object.'''
         arches = Arch.objects.filter(agnostic=False)
-        repos = Repo.objects.filter(staging=False)
+        repos = Repo.objects.all()
         arch_path_map = dict((arch, None) for arch in arches)
         all_paths = set()
         total_paths = 0
-- 
cgit v1.2.3-2-g168b


From 5228cb5f584f076e547e1d0af695c08975801d2f Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Fri, 12 Oct 2012 11:39:16 -0500
Subject: reporead: don't print full backtrace if unnecessary

In the architecture agnostic case, this error is much more likely to
happen, so printing it like an error message is deceiving.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index ce5c8cb7..a1e77b49 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -393,9 +393,12 @@ def db_update(archname, reponame, pkgs, force=False):
                 populate_pkg(dbpkg, pkg, timestamp=now())
                 Update.objects.log_update(None, dbpkg)
         except IntegrityError:
-            logger.warning("Could not add package %s; "
-                    "not fatal if another thread beat us to it.",
-                    pkg.name, exc_info=True)
+            if architecture.agnostic:
+                logger.warning("Could not add package %s; "
+                        "not fatal if another thread beat us to it.",
+                        pkg.name)
+            else:
+                logger.exception("Could not add package %s", pkg.name)
 
     # packages in database and not in syncdb (remove from database)
     for pkgname in (dbset - syncset):
-- 
cgit v1.2.3-2-g168b


From 6dd4d54bb0adbbb0f8c2b1beaa92b7a58971cf88 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Fri, 16 Nov 2012 16:20:11 -0600
Subject: Use Python 2.7 dictionary comprehension syntax

Rather than the old idiom of dict((k, v) for <> in <>).

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py         | 4 ++--
 devel/management/commands/reporead_inotify.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index a1e77b49..3d4e6375 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -377,7 +377,7 @@ def db_update(archname, reponame, pkgs, force=False):
     # This makes our inner loop where we find packages by name *way* more
     # efficient by not having to go to the database for each package to
     # SELECT them by name.
-    dbdict = dict((dbpkg.pkgname, dbpkg) for dbpkg in dbpkgs)
+    dbdict = {dbpkg.pkgname: dbpkg for dbpkg in dbpkgs}
 
     dbset = set(dbdict.keys())
     syncset = set([pkg.name for pkg in pkgs])
@@ -446,7 +446,7 @@ def filesonly_update(archname, reponame, pkgs, force=False):
     """
     logger.info('Updating files for %s (%s)', reponame, archname)
     dbpkgs = update_common(archname, reponame, pkgs, False)
-    dbdict = dict((dbpkg.pkgname, dbpkg) for dbpkg in dbpkgs)
+    dbdict = {dbpkg.pkgname: dbpkg for dbpkg in dbpkgs}
     dbset = set(dbdict.keys())
 
     for pkg in (pkg for pkg in pkgs if pkg.name in dbset):
diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py
index c74762eb..16b3869c 100644
--- a/devel/management/commands/reporead_inotify.py
+++ b/devel/management/commands/reporead_inotify.py
@@ -69,7 +69,7 @@ class Command(BaseCommand):
         finally builds and returns a notifier object.'''
         arches = Arch.objects.filter(agnostic=False)
         repos = Repo.objects.all()
-        arch_path_map = dict((arch, None) for arch in arches)
+        arch_path_map = {arch: None for arch in arches}
         all_paths = set()
         total_paths = 0
         for arch in arches:
-- 
cgit v1.2.3-2-g168b


From 9e9157d0a8cbf9ea076231e438fb30f58bff8e29 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Fri, 16 Nov 2012 16:37:31 -0600
Subject: Use python set comprehension syntax supported in 2.7

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/import_signatures.py | 4 ++--
 devel/management/commands/reporead.py          | 2 +-
 devel/management/commands/reporead_inotify.py  | 2 +-
 devel/views.py                                 | 4 ++--
 4 files changed, 6 insertions(+), 6 deletions(-)

(limited to 'devel')

diff --git a/devel/management/commands/import_signatures.py b/devel/management/commands/import_signatures.py
index ce1aba90..da1397ca 100644
--- a/devel/management/commands/import_signatures.py
+++ b/devel/management/commands/import_signatures.py
@@ -98,8 +98,8 @@ def import_signatures(keyring):
 
     # now prune the data down to what we actually want.
     # prune edges not in nodes, remove duplicates, and self-sigs
-    pruned_edges = set(edge for edge in edges
-            if edge.signer in nodes and edge.signer != edge.signee)
+    pruned_edges = {edge for edge in edges
+            if edge.signer in nodes and edge.signer != edge.signee}
 
     logger.info("creating or finding %d signatures", len(pruned_edges))
     created_ct = updated_ct = 0
diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index 3d4e6375..981c4dce 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -380,7 +380,7 @@ def db_update(archname, reponame, pkgs, force=False):
     dbdict = {dbpkg.pkgname: dbpkg for dbpkg in dbpkgs}
 
     dbset = set(dbdict.keys())
-    syncset = set([pkg.name for pkg in pkgs])
+    syncset = {pkg.name for pkg in pkgs}
 
     in_sync_not_db = syncset - dbset
     logger.info("%d packages in sync not db", len(in_sync_not_db))
diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py
index 16b3869c..04f65764 100644
--- a/devel/management/commands/reporead_inotify.py
+++ b/devel/management/commands/reporead_inotify.py
@@ -77,7 +77,7 @@ class Command(BaseCommand):
                     for repo in repos)
             # take a python format string and generate all unique combinations
             # of directories from it; using set() ensures we filter it down
-            paths = set(self.path_template % values for values in combos)
+            paths = {self.path_template % values for values in combos}
             total_paths += len(paths)
             all_paths |= paths
             arch_path_map[arch] = paths
diff --git a/devel/views.py b/devel/views.py
index 083665d9..7d5947d1 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -277,8 +277,8 @@ def report(request, report_name, username=None):
     else:
         raise Http404
 
-    arches = set(pkg.arch for pkg in packages)
-    repos = set(pkg.repo for pkg in packages)
+    arches = {pkg.arch for pkg in packages}
+    repos = {pkg.repo for pkg in packages}
     context = {
         'all_maintainers': maints,
         'title': title,
-- 
cgit v1.2.3-2-g168b


From 4c699119820dfd060de6a0385e549f3397053548 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Tue, 4 Dec 2012 21:59:29 -0600
Subject: get_latest_by cleanups

Fix some that referenced non-existent attributes, and add the attribute
to other models.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/models.py | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'devel')

diff --git a/devel/models.py b/devel/models.py
index 9b6f07a7..f30bba85 100644
--- a/devel/models.py
+++ b/devel/models.py
@@ -50,6 +50,7 @@ class UserProfile(models.Model):
 
     class Meta:
         db_table = 'user_profiles'
+        get_latest_by = 'last_modified'
         verbose_name = 'Additional Profile Data'
         verbose_name_plural = 'Additional Profile Data'
 
@@ -80,6 +81,7 @@ class MasterKey(models.Model):
 
     class Meta:
         ordering = ('created',)
+        get_latest_by = 'created'
 
     def __unicode__(self):
         return u'%s, created %s' % (
@@ -94,6 +96,8 @@ class PGPSignature(models.Model):
     valid = models.BooleanField(default=True)
 
     class Meta:
+        ordering = ('signer', 'signee')
+        get_latest_by = 'created'
         verbose_name = 'PGP signature'
 
     def __unicode__(self):
-- 
cgit v1.2.3-2-g168b


From e68a5073a6e8b9473f726734e0b51fdb0a42c14b Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Thu, 27 Dec 2012 17:02:40 -0600
Subject: Fix "RuntimeWarning: DateTimeField received a naive datetime"
 warnings

When running tests, we can find old migrations that didn't use datetime
objects with timezones attached.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/migrations/0008_auto__add_field_userprofile_last_modified.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'devel')

diff --git a/devel/migrations/0008_auto__add_field_userprofile_last_modified.py b/devel/migrations/0008_auto__add_field_userprofile_last_modified.py
index 2695987a..08972e1b 100644
--- a/devel/migrations/0008_auto__add_field_userprofile_last_modified.py
+++ b/devel/migrations/0008_auto__add_field_userprofile_last_modified.py
@@ -3,12 +3,14 @@ import datetime
 from south.db import db
 from south.v2 import SchemaMigration
 from django.db import models
+from pytz import utc
 
 
 class Migration(SchemaMigration):
     def forwards(self, orm):
+        default = datetime.datetime(2000, 1, 1, 0, 0).replace(tzinfo=utc)
         db.add_column('user_profiles', 'last_modified',
-                      self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime(2000, 1, 1, 0, 0)),
+                      self.gf('django.db.models.fields.DateTimeField')(default=default),
                       keep_default=False)
 
     def backwards(self, orm):
-- 
cgit v1.2.3-2-g168b


From bf4385a26c1b6f07bf9bdcddf7160b5eb4a71d9a Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Thu, 27 Dec 2012 21:13:56 -0600
Subject: Move the body of set_last_modified to main/utils

Instead of having multiple methods, move this into our single 'created'
setter method. If the 'last_modified' property is present, we now update
it accordingly when saving any model with this signal attached.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/models.py | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

(limited to 'devel')

diff --git a/devel/models.py b/devel/models.py
index f30bba85..5f0a8318 100644
--- a/devel/models.py
+++ b/devel/models.py
@@ -8,7 +8,7 @@ from django.utils.timezone import now
 from django_countries import CountryField
 
 from .fields import PGPKeyField
-from main.utils import make_choice
+from main.utils import make_choice, set_created_field
 
 
 class UserProfile(models.Model):
@@ -104,17 +104,7 @@ class PGPSignature(models.Model):
         return u'%s → %s' % (self.signer, self.signee)
 
 
-def set_last_modified(sender, **kwargs):
-    '''This will set the 'last_modified' field on the user profile to the
-    current UTC time when either the profile is updated.  For use as a pre_save
-    signal handler.'''
-    obj = kwargs['instance']
-    if hasattr(obj, 'last_modified'):
-        obj.last_modified = now()
-
-
-# connect signals needed to keep cache in line with reality
-pre_save.connect(set_last_modified, sender=UserProfile,
+pre_save.connect(set_created_field, sender=UserProfile,
         dispatch_uid="devel.models")
 
 # vim: set ts=4 sw=4 et:
-- 
cgit v1.2.3-2-g168b


From c8ece67cec9c421ac0c711554edd34f022623b45 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Fri, 28 Dec 2012 00:27:20 -0600
Subject: Convert to using new todolist models everywhere

This is a rather widespread set of changes converting usage to the new
todo list and todo list package model recently introduced. The data
migration is not included in this commit. After this commit, the old
model should no longer be referenced anywhere.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/views.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

(limited to 'devel')

diff --git a/devel/views.py b/devel/views.py
index 7d5947d1..e01590a0 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -20,11 +20,12 @@ from django.utils.http import http_date
 from django.utils.timezone import now
 
 from .forms import ProfileForm, UserProfileForm, NewUserForm
-from main.models import Package, PackageFile, TodolistPkg
+from main.models import Package, PackageFile
 from main.models import Arch, Repo
 from news.models import News
 from packages.models import PackageRelation, Signoff, FlagRequest, Depend
 from packages.utils import get_signoff_groups
+from todolists.models import TodolistPackage
 from todolists.utils import get_annotated_todolists
 from .utils import get_annotated_maintainers, UserFinder
 
@@ -41,10 +42,11 @@ def index(request):
     flagged = Package.objects.normal().filter(
             flag_date__isnull=False, pkgbase__in=inner_q).order_by('pkgname')
 
-    todopkgs = TodolistPkg.objects.select_related(
-            'pkg', 'pkg__arch', 'pkg__repo').filter(complete=False)
-    todopkgs = todopkgs.filter(pkg__pkgbase__in=inner_q).order_by(
-            'list__name', 'pkg__pkgname')
+    todopkgs = TodolistPackage.objects.select_related(
+            'todolist', 'pkg', 'arch', 'repo').exclude(
+            status=TodolistPackage.COMPLETE)
+    todopkgs = todopkgs.filter(pkgbase__in=inner_q).order_by(
+            'todolist__name', 'pkgname')
 
     todolists = get_annotated_todolists(incomplete_only=True)
 
-- 
cgit v1.2.3-2-g168b


From 04f23a040a839f4989fdc83afe0f5ad4f72224be Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Mon, 31 Dec 2012 10:17:22 -0600
Subject: Add 'created' field to packages model

This will be used to eventually implement the UI side of FS#13441, but
to do that, we first need the data.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index 981c4dce..e00e54c3 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -387,10 +387,12 @@ def db_update(archname, reponame, pkgs, force=False):
     # packages in syncdb and not in database (add to database)
     for pkg in (pkg for pkg in pkgs if pkg.name in in_sync_not_db):
         logger.info("Adding package %s", pkg.name)
-        dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository)
+        timestamp = now()
+        dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository,
+                created=timestamp)
         try:
             with transaction.commit_on_success():
-                populate_pkg(dbpkg, pkg, timestamp=now())
+                populate_pkg(dbpkg, pkg, timestamp=timestamp)
                 Update.objects.log_update(None, dbpkg)
         except IntegrityError:
             if architecture.agnostic:
-- 
cgit v1.2.3-2-g168b


From 7952fe0ede3a5a68a64f05eccb180194394652f3 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Mon, 31 Dec 2012 11:31:35 -0600
Subject: Mark todolist packages as removed rather than deleting them

This makes it easier to see the progression of a todolist and its
contents easier since we are no longer losing the data.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'devel')

diff --git a/devel/views.py b/devel/views.py
index e01590a0..90839847 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -44,7 +44,7 @@ def index(request):
 
     todopkgs = TodolistPackage.objects.select_related(
             'todolist', 'pkg', 'arch', 'repo').exclude(
-            status=TodolistPackage.COMPLETE)
+            status=TodolistPackage.COMPLETE).filter(removed__isnull=True)
     todopkgs = todopkgs.filter(pkgbase__in=inner_q).order_by(
             'todolist__name', 'pkgname')
 
-- 
cgit v1.2.3-2-g168b


From 3d23df0b661de5ccd5096dda22abcfb161e8b1b4 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Mon, 31 Dec 2012 11:47:14 -0600
Subject: Fix case of devel user profile verbose name

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/models.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'devel')

diff --git a/devel/models.py b/devel/models.py
index 5f0a8318..6689ca3d 100644
--- a/devel/models.py
+++ b/devel/models.py
@@ -51,8 +51,8 @@ class UserProfile(models.Model):
     class Meta:
         db_table = 'user_profiles'
         get_latest_by = 'last_modified'
-        verbose_name = 'Additional Profile Data'
-        verbose_name_plural = 'Additional Profile Data'
+        verbose_name = 'additional profile data'
+        verbose_name_plural = 'additional profile data'
 
     def get_absolute_url(self):
         # TODO: this is disgusting. find a way to consolidate this logic with
-- 
cgit v1.2.3-2-g168b


From af32c23768c7537f19e0613525579208b4f44eb4 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Tue, 15 Jan 2013 20:49:56 -0600
Subject: Handle connection and transaction more properly in reporead

A few minor things are fixed here. One is PostgreSQL, and more
specifically pgbouncer, don't like it when the connection is closed
after psycopg2 has started an implicit transaction even for read-only
queries. Ensure we call commit as our last database action in all cases.

The other is related- Django in management commands doesn't ever call
close on any database connection you may have been using, so PostgreSQL
gets mad about this fact and logs a message saying such. Close the
connection explicitly when we are done with it to play nice.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py         |  1 +
 devel/management/commands/reporead_inotify.py | 15 +++++++++------
 2 files changed, 10 insertions(+), 6 deletions(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index e00e54c3..ab0efeed 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -589,6 +589,7 @@ def read_repo(primary_arch, repo_file, options):
         else:
             db_update(arch, repo, packages_arches[arch], force)
     logger.info('Finished database updates for %s.', repo_file)
+    connection.commit()
     connection.close()
     return 0
 
diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py
index 04f65764..8c1e47bf 100644
--- a/devel/management/commands/reporead_inotify.py
+++ b/devel/management/commands/reporead_inotify.py
@@ -23,7 +23,7 @@ import threading
 import time
 
 from django.core.management.base import BaseCommand, CommandError
-from django.db import connection
+from django.db import connection, transaction
 
 from main.models import Arch, Repo
 from .reporead import read_repo
@@ -53,6 +53,11 @@ class Command(BaseCommand):
         self.path_template = path_template
 
         notifier = self.setup_notifier()
+        # this thread is done using the database; all future access is done in
+        # the spawned read_repo() processes, so close the otherwise completely
+        # idle connection.
+        connection.close()
+
         logger.info('Entering notifier loop')
         notifier.loop()
 
@@ -61,14 +66,17 @@ class Command(BaseCommand):
             if hasattr(thread, 'cancel'):
                 thread.cancel()
 
+    @transaction.commit_on_success
     def setup_notifier(self):
         '''Set up and configure the inotify machinery and logic.
         This takes the provided or default path_template and builds a list of
         directories we need to watch for database updates. It then validates
         and passes these on to the various pyinotify pieces as necessary and
         finally builds and returns a notifier object.'''
+        transaction.commit_manually()
         arches = Arch.objects.filter(agnostic=False)
         repos = Repo.objects.all()
+        transaction.set_dirty()
         arch_path_map = {arch: None for arch in arches}
         all_paths = set()
         total_paths = 0
@@ -91,11 +99,6 @@ class Command(BaseCommand):
             raise CommandError('path template did not uniquely '
                     'determine architecture for each file')
 
-        # this thread is done using the database; all future access is done in
-        # the spawned read_repo() processes, so close the otherwise completely
-        # idle connection.
-        connection.close()
-
         # A proper atomic replacement of the database as done by rsync is type
         # IN_MOVED_TO. repo-add/remove will finish with a IN_CLOSE_WRITE.
         mask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO
-- 
cgit v1.2.3-2-g168b


From 9da8a63dd476fe3607a68a028655c9f9d0fee163 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Sun, 3 Feb 2013 13:55:25 -0600
Subject: Add DeveloperKey model

We're starting to see developers use subkeys of their primary key to
sign packages, which we aren't handling well in the web interface. These
subkeys show up as unknown, which isn't strictly true. Start the process
of being able to handle these keys by adding a model that will store all
known keys and subkeys and the relationships among them, as well as
which developer owns each.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/admin.py                                  |  12 ++-
 devel/migrations/0009_auto__add_developerkey.py | 126 ++++++++++++++++++++++++
 devel/models.py                                 |  15 ++-
 3 files changed, 150 insertions(+), 3 deletions(-)
 create mode 100644 devel/migrations/0009_auto__add_developerkey.py

(limited to 'devel')

diff --git a/devel/admin.py b/devel/admin.py
index 5a704c0b..971933b7 100644
--- a/devel/admin.py
+++ b/devel/admin.py
@@ -2,7 +2,7 @@ from django.contrib import admin
 from django.contrib.auth.admin import UserAdmin
 from django.contrib.auth.models import User
 
-from .models import UserProfile, MasterKey, PGPSignature
+from .models import UserProfile, MasterKey, DeveloperKey, PGPSignature
 
 
 class UserProfileInline(admin.StackedInline):
@@ -17,7 +17,14 @@ class UserProfileAdmin(UserAdmin):
 
 class MasterKeyAdmin(admin.ModelAdmin):
     list_display = ('pgp_key', 'owner', 'created', 'revoker', 'revoked')
-    search_fields = ('pgp_key', 'owner', 'revoker')
+    search_fields = ('pgp_key', 'owner__username', 'revoker__username')
+    date_hierarchy = 'created'
+
+
+class DeveloperKeyAdmin(admin.ModelAdmin):
+    list_display = ('key', 'parent', 'owner', 'created', 'expires', 'revoked')
+    search_fields = ('key', 'owner__username')
+    list_filter = ('owner',)
     date_hierarchy = 'created'
 
 
@@ -32,6 +39,7 @@ admin.site.unregister(User)
 admin.site.register(User, UserProfileAdmin)
 
 admin.site.register(MasterKey, MasterKeyAdmin)
+admin.site.register(DeveloperKey, DeveloperKeyAdmin)
 admin.site.register(PGPSignature, PGPSignatureAdmin)
 
 # vim: set ts=4 sw=4 et:
diff --git a/devel/migrations/0009_auto__add_developerkey.py b/devel/migrations/0009_auto__add_developerkey.py
new file mode 100644
index 00000000..60d3f7b8
--- /dev/null
+++ b/devel/migrations/0009_auto__add_developerkey.py
@@ -0,0 +1,126 @@
+# -*- coding: utf-8 -*-
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        db.create_table('devel_developerkey', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('owner', self.gf('django.db.models.fields.related.ForeignKey')(related_name='all_keys', null=True, to=orm['auth.User'])),
+            ('key', self.gf('devel.fields.PGPKeyField')(unique=True, max_length=40)),
+            ('created', self.gf('django.db.models.fields.DateTimeField')()),
+            ('expires', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)),
+            ('revoked', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)),
+            ('parent', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['devel.DeveloperKey'], null=True, on_delete=models.SET_NULL)),
+        ))
+        db.send_create_signal('devel', ['DeveloperKey'])
+
+    def backwards(self, orm):
+        db.delete_table('devel_developerkey')
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'devel.developerkey': {
+            'Meta': {'object_name': 'DeveloperKey'},
+            'created': ('django.db.models.fields.DateTimeField', [], {}),
+            'expires': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'key': ('devel.fields.PGPKeyField', [], {'unique': 'True', 'max_length': '40'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'all_keys'", 'null': 'True', 'to': "orm['auth.User']"}),
+            'parent': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['devel.DeveloperKey']", 'null': 'True', 'on_delete': 'models.SET_NULL'}),
+            'revoked': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'})
+        },
+        'devel.masterkey': {
+            'Meta': {'ordering': "('created',)", 'object_name': 'MasterKey'},
+            'created': ('django.db.models.fields.DateField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'masterkey_owner'", 'to': "orm['auth.User']"}),
+            'pgp_key': ('devel.fields.PGPKeyField', [], {'max_length': '40'}),
+            'revoked': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'revoker': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'masterkey_revoker'", 'to': "orm['auth.User']"})
+        },
+        'devel.pgpsignature': {
+            'Meta': {'ordering': "('signer', 'signee')", 'object_name': 'PGPSignature'},
+            'created': ('django.db.models.fields.DateField', [], {}),
+            'expires': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'signee': ('devel.fields.PGPKeyField', [], {'max_length': '40'}),
+            'signer': ('devel.fields.PGPKeyField', [], {'max_length': '40'}),
+            'valid': ('django.db.models.fields.BooleanField', [], {'default': 'True'})
+        },
+        'devel.userprofile': {
+            'Meta': {'object_name': 'UserProfile', 'db_table': "'user_profiles'"},
+            'alias': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'allowed_repos': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['main.Repo']", 'symmetrical': 'False', 'blank': 'True'}),
+            'country': ('django_countries.fields.CountryField', [], {'max_length': '2', 'blank': 'True'}),
+            'favorite_distros': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'interests': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'languages': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'last_modified': ('django.db.models.fields.DateTimeField', [], {}),
+            'latin_name': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'location': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'notify': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'occupation': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}),
+            'other_contact': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'pgp_key': ('devel.fields.PGPKeyField', [], {'max_length': '40', 'null': 'True', 'blank': 'True'}),
+            'picture': ('django.db.models.fields.files.FileField', [], {'default': "'devs/silhouette.png'", 'max_length': '100'}),
+            'public_email': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'roles': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
+            'time_zone': ('django.db.models.fields.CharField', [], {'default': "'UTC'", 'max_length': '100'}),
+            'user': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'userprofile'", 'unique': 'True', 'to': "orm['auth.User']"}),
+            'website': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'yob': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'})
+        },
+        'main.repo': {
+            'Meta': {'ordering': "('name',)", 'object_name': 'Repo', 'db_table': "'repos'"},
+            'bugs_category': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}),
+            'bugs_project': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+            'staging': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'svn_root': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'testing': ('django.db.models.fields.BooleanField', [], {'default': 'False'})
+        }
+    }
+
+    complete_apps = ['devel']
diff --git a/devel/models.py b/devel/models.py
index 6689ca3d..67de40a6 100644
--- a/devel/models.py
+++ b/devel/models.py
@@ -68,7 +68,6 @@ class UserProfile(models.Model):
         return '/%s/#%s' % (prefix, self.user.username)
 
 
-
 class MasterKey(models.Model):
     owner = models.ForeignKey(User, related_name='masterkey_owner',
         help_text="The developer holding this master key")
@@ -88,6 +87,20 @@ class MasterKey(models.Model):
                 self.owner.get_full_name(), self.created)
 
 
+class DeveloperKey(models.Model):
+    owner = models.ForeignKey(User, related_name='all_keys', null=True,
+        help_text="The developer this key belongs to")
+    key = PGPKeyField(max_length=40, verbose_name="PGP key fingerprint",
+			unique=True)
+    created = models.DateTimeField()
+    expires = models.DateTimeField(null=True, blank=True)
+    revoked = models.DateTimeField(null=True, blank=True)
+    parent = models.ForeignKey('self', null=True, on_delete=models.SET_NULL)
+
+    def __unicode__(self):
+        return self.key
+
+
 class PGPSignature(models.Model):
     signer = PGPKeyField(max_length=40, verbose_name="Signer key fingerprint")
     signee = PGPKeyField(max_length=40, verbose_name="Signee key fingerprint")
-- 
cgit v1.2.3-2-g168b


From 7e6279057a57ef44c11349e594ad392fbfce0098 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Sun, 3 Feb 2013 14:26:10 -0600
Subject: Add new pgp_import command; replaces import_signatures

This command now imports keys, subkeys, and signatures of those keys &
subkeys. This will allow us to actually match developers with their
packages signed by subkeys rather than the primary key.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/import_signatures.py | 123 -------------
 devel/management/commands/pgp_import.py        | 241 +++++++++++++++++++++++++
 2 files changed, 241 insertions(+), 123 deletions(-)
 delete mode 100644 devel/management/commands/import_signatures.py
 create mode 100644 devel/management/commands/pgp_import.py

(limited to 'devel')

diff --git a/devel/management/commands/import_signatures.py b/devel/management/commands/import_signatures.py
deleted file mode 100644
index da1397ca..00000000
--- a/devel/management/commands/import_signatures.py
+++ /dev/null
@@ -1,123 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-import_signatures command
-
-Import signatures from a given GPG keyring.
-
-Usage: ./manage.py generate_keyring <keyring_path>
-"""
-
-from collections import namedtuple
-from datetime import datetime
-import logging
-import subprocess
-import sys
-
-from django.core.management.base import BaseCommand, CommandError
-from django.db import transaction
-
-from devel.models import PGPSignature
-
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s -> %(levelname)s: %(message)s',
-    datefmt='%Y-%m-%d %H:%M:%S',
-    stream=sys.stderr)
-logger = logging.getLogger()
-
-class Command(BaseCommand):
-    args = "<keyring_path>"
-    help = "Import signatures from a given GPG keyring."
-
-    def handle(self, *args, **options):
-        v = int(options.get('verbosity', None))
-        if v == 0:
-            logger.level = logging.ERROR
-        elif v == 1:
-            logger.level = logging.INFO
-        elif v == 2:
-            logger.level = logging.DEBUG
-
-        if len(args) < 1:
-            raise CommandError("keyring_path must be provided")
-
-        import_signatures(args[0])
-
-
-SignatureData = namedtuple('SignatureData',
-        ('signer', 'signee', 'created', 'expires', 'valid'))
-
-
-def get_date(epoch_string):
-    '''Convert a epoch string into a python 'date' object (not datetime).'''
-    return datetime.utcfromtimestamp(int(epoch_string)).date()
-
-
-def parse_sigdata(data):
-    nodes = {}
-    edges = []
-    current_pubkey = None
-
-    # parse all of the output from our successful GPG command
-    logger.info("parsing command output")
-    for line in data.split('\n'):
-        parts = line.split(':')
-        if parts[0] == 'pub':
-            current_pubkey = parts[4]
-            nodes[current_pubkey] = None
-        if parts[0] == 'uid':
-            uid = parts[9]
-            # only set uid if this is the first one encountered
-            if nodes[current_pubkey] is None:
-                nodes[current_pubkey] = uid
-        if parts[0] == 'sig':
-            signer = parts[4]
-            created = get_date(parts[5])
-            expires = None
-            if parts[6]:
-                expires = get_date(parts[6])
-            valid = parts[1] != '-'
-            edge = SignatureData(signer, current_pubkey,
-                    created, expires, valid)
-            edges.append(edge)
-
-    return nodes, edges
-
-
-def import_signatures(keyring):
-    gpg_cmd = ["gpg", "--no-default-keyring", "--keyring", keyring,
-            "--list-sigs", "--with-colons", "--fixed-list-mode"]
-    logger.info("running command: %r", gpg_cmd)
-    proc = subprocess.Popen(gpg_cmd, stdout=subprocess.PIPE)
-    outdata, errdata = proc.communicate()
-    if proc.returncode != 0:
-        logger.error(errdata)
-        raise subprocess.CalledProcessError(proc.returncode, gpg_cmd)
-
-    nodes, edges = parse_sigdata(outdata)
-
-    # now prune the data down to what we actually want.
-    # prune edges not in nodes, remove duplicates, and self-sigs
-    pruned_edges = {edge for edge in edges
-            if edge.signer in nodes and edge.signer != edge.signee}
-
-    logger.info("creating or finding %d signatures", len(pruned_edges))
-    created_ct = updated_ct = 0
-    with transaction.commit_on_success():
-        for edge in pruned_edges:
-            sig, created = PGPSignature.objects.get_or_create(
-                    signer=edge.signer, signee=edge.signee,
-                    created=edge.created, expires=edge.expires,
-                    defaults={ 'valid': edge.valid })
-            if sig.valid != edge.valid:
-                sig.valid = edge.valid
-                sig.save()
-                updated_ct = 1
-            if created:
-                created_ct += 1
-
-    sig_ct = PGPSignature.objects.all().count()
-    logger.info("%d total signatures in database", sig_ct)
-    logger.info("created %d, updated %d signatures", created_ct, updated_ct)
-
-# vim: set ts=4 sw=4 et:
diff --git a/devel/management/commands/pgp_import.py b/devel/management/commands/pgp_import.py
new file mode 100644
index 00000000..10e6cfcb
--- /dev/null
+++ b/devel/management/commands/pgp_import.py
@@ -0,0 +1,241 @@
+# -*- coding: utf-8 -*-
+"""
+pgp_import command
+
+Import keys and signatures from a given GPG keyring.
+
+Usage: ./manage.py pgp_import <keyring_path>
+"""
+
+from collections import namedtuple, OrderedDict
+from datetime import datetime
+import logging
+from pytz import utc
+import subprocess
+import sys
+
+from django.core.management.base import BaseCommand, CommandError
+from django.db import transaction
+
+from devel.models import DeveloperKey, PGPSignature
+from devel.utils import UserFinder
+
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s -> %(levelname)s: %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S',
+    stream=sys.stderr)
+logger = logging.getLogger()
+
+class Command(BaseCommand):
+    args = "<keyring_path>"
+    help = "Import keys and signatures from a given GPG keyring."
+
+    def handle(self, *args, **options):
+        v = int(options.get('verbosity', None))
+        if v == 0:
+            logger.level = logging.ERROR
+        elif v == 1:
+            logger.level = logging.INFO
+        elif v == 2:
+            logger.level = logging.DEBUG
+
+        if len(args) < 1:
+            raise CommandError("keyring_path must be provided")
+
+        import_keys(args[0])
+        import_signatures(args[0])
+
+
+def get_date(epoch_string):
+    '''Convert a epoch string into a python 'date' object (not datetime).'''
+    if not epoch_string:
+        return None
+    return datetime.utcfromtimestamp(int(epoch_string)).date()
+
+
+def get_datetime(epoch_string):
+    '''Convert a epoch string into a python 'datetime' object.'''
+    if not epoch_string:
+        return None
+    return datetime.utcfromtimestamp(int(epoch_string)).replace(tzinfo=utc)
+
+
+def call_gpg(keyring, *args):
+    # GPG is stupid and interprets any filename without path portion as being
+    # in ~/.gnupg/. Fake it out if we just get a bare filename.
+    if '/' not in keyring:
+        keyring = './%s' % keyring
+    gpg_cmd = ["gpg2", "--no-default-keyring", "--keyring", keyring,
+            "--with-colons", "--fixed-list-mode"]
+    gpg_cmd.extend(args)
+    logger.info("running command: %s", ' '.join(gpg_cmd))
+    proc = subprocess.Popen(gpg_cmd, stdout=subprocess.PIPE)
+    outdata, errdata = proc.communicate()
+    if proc.returncode != 0:
+        logger.error(errdata)
+        raise subprocess.CalledProcessError(proc.returncode, gpg_cmd)
+    return outdata
+
+
+class KeyData(object):
+    def __init__(self, key, created, expires):
+        self.key = key
+        self.created = get_datetime(created)
+        self.expires = get_datetime(expires)
+        self.parent = None
+        self.revoked = None
+        self.db_id = None
+
+
+def parse_keydata(data):
+    keys = OrderedDict()
+    current_pubkey = None
+
+    # parse all of the output from our successful GPG command
+    logger.info("parsing command output")
+    for line in data.split('\n'):
+        parts = line.split(':')
+        if parts[0] == 'pub':
+            key = parts[4]
+            current_pubkey = key
+            keys[key] = KeyData(key, parts[5], parts[6])
+            node = parts[0]
+        elif parts[0] == 'sub':
+            key = parts[4]
+            keys[key] = KeyData(key, parts[5], parts[6])
+            keys[key].parent = current_pubkey
+            node = parts[0]
+        elif parts[0] == 'uid':
+            node = parts[0]
+        elif parts[0] == 'rev' and node in ('pub', 'sub'):
+            keys[current_pubkey].revoked = get_datetime(parts[5])
+
+    return keys
+
+
+def find_key_owner(key, keys, finder):
+    '''Recurse up the chain, looking for an owner.'''
+    if key is None:
+        return None
+    owner = finder.find_by_pgp_key(key.key)
+    if owner:
+        return owner
+    if key.parent:
+        return find_key_owner(keys[key.parent], keys, finder)
+    return None
+
+
+def import_keys(keyring):
+    outdata = call_gpg(keyring, "--list-sigs")
+    keydata = parse_keydata(outdata)
+
+    logger.info("creating or finding %d keys", len(keydata))
+    created_ct = updated_ct = 0
+    with transaction.commit_on_success():
+        finder = UserFinder()
+        # we are dependent on parents coming before children; parse_keydata
+        # uses an OrderedDict to ensure this is the case.
+        for data in keydata.values():
+            parent_id = None
+            if data.parent:
+                parent_data = keydata.get(data.parent, None)
+                if parent_data:
+                    parent_id = parent_data.db_id
+            other = {
+                'expires': data.expires,
+                'revoked': data.revoked,
+                'parent_id': parent_id,
+            }
+            dkey, created = DeveloperKey.objects.get_or_create(
+                    key=data.key, created=data.created, defaults=other)
+            data.db_id = dkey.id
+
+            # set or update any additional data we might need to
+            needs_save = False
+            if created:
+                created_ct += 1
+            else:
+                for k, v in other.items():
+                    if getattr(dkey, k) != v:
+                        setattr(dkey, k, v)
+                        needs_save = True
+            if dkey.owner_id is None:
+                owner = find_key_owner(data, keydata, finder)
+                if owner is not None:
+                    dkey.owner = owner
+                    needs_save = True
+            if needs_save:
+                dkey.save()
+                updated_ct += 1
+
+    key_ct = DeveloperKey.objects.all().count()
+    logger.info("%d total keys in database", key_ct)
+    logger.info("created %d, updated %d keys", created_ct, updated_ct)
+
+
+SignatureData = namedtuple('SignatureData',
+        ('signer', 'signee', 'created', 'expires', 'valid'))
+
+
+def parse_sigdata(data):
+    nodes = {}
+    edges = []
+    current_pubkey = None
+
+    # parse all of the output from our successful GPG command
+    logger.info("parsing command output")
+    for line in data.split('\n'):
+        parts = line.split(':')
+        if parts[0] == 'pub':
+            current_pubkey = parts[4]
+            nodes[current_pubkey] = None
+        if parts[0] == 'uid':
+            uid = parts[9]
+            # only set uid if this is the first one encountered
+            if nodes[current_pubkey] is None:
+                nodes[current_pubkey] = uid
+        if parts[0] == 'sig':
+            signer = parts[4]
+            created = get_date(parts[5])
+            expires = None
+            if parts[6]:
+                expires = get_date(parts[6])
+            valid = parts[1] != '-'
+            edge = SignatureData(signer, current_pubkey,
+                    created, expires, valid)
+            edges.append(edge)
+
+    return nodes, edges
+
+
+def import_signatures(keyring):
+    outdata = call_gpg(keyring, "--list-sigs")
+    nodes, edges = parse_sigdata(outdata)
+
+    # now prune the data down to what we actually want.
+    # prune edges not in nodes, remove duplicates, and self-sigs
+    pruned_edges = {edge for edge in edges
+            if edge.signer in nodes and edge.signer != edge.signee}
+
+    logger.info("creating or finding %d signatures", len(pruned_edges))
+    created_ct = updated_ct = 0
+    with transaction.commit_on_success():
+        for edge in pruned_edges:
+            sig, created = PGPSignature.objects.get_or_create(
+                    signer=edge.signer, signee=edge.signee,
+                    created=edge.created, expires=edge.expires,
+                    defaults={ 'valid': edge.valid })
+            if sig.valid != edge.valid:
+                sig.valid = edge.valid
+                sig.save()
+                updated_ct = 1
+            if created:
+                created_ct += 1
+
+    sig_ct = PGPSignature.objects.all().count()
+    logger.info("%d total signatures in database", sig_ct)
+    logger.info("created %d, updated %d signatures", created_ct, updated_ct)
+
+# vim: set ts=4 sw=4 et:
-- 
cgit v1.2.3-2-g168b


From 508d06af810c787b2644331444279407ccfa27af Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Sun, 3 Feb 2013 15:08:21 -0600
Subject: Use DeveloperKey model on package page and reports

This introduces the new model to the package page so subkey signings
show up as attributed to the original developer. We also teach the
mismatched signatures report to recognize all keys and subkeys of a
given developer, cutting down on some of the bogus results.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/views.py | 33 +++++++++++++++++++++++----------
 1 file changed, 23 insertions(+), 10 deletions(-)

(limited to 'devel')

diff --git a/devel/views.py b/devel/views.py
index 90839847..ff1dec12 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -20,6 +20,7 @@ from django.utils.http import http_date
 from django.utils.timezone import now
 
 from .forms import ProfileForm, UserProfileForm, NewUserForm
+from .models import DeveloperKey
 from main.models import Package, PackageFile
 from main.models import Arch, Repo
 from news.models import News
@@ -27,7 +28,7 @@ from packages.models import PackageRelation, Signoff, FlagRequest, Depend
 from packages.utils import get_signoff_groups
 from todolists.models import TodolistPackage
 from todolists.utils import get_annotated_todolists
-from .utils import get_annotated_maintainers, UserFinder
+from .utils import get_annotated_maintainers
 
 
 @login_required
@@ -262,18 +263,30 @@ def report(request, report_name, username=None):
         names = [ 'Signature Date', 'Signed By', 'Packager' ]
         attrs = [ 'sig_date', 'sig_by', 'packager' ]
         cutoff = timedelta(hours=24)
-        finder = UserFinder()
         filtered = []
-        packages = packages.filter(pgp_signature__isnull=False)
+        packages = packages.select_related(
+                'arch', 'repo', 'packager').filter(pgp_signature__isnull=False)
+        known_keys = DeveloperKey.objects.select_related(
+                'owner').filter(owner__isnull=False)
+        known_keys = {dk.key: dk for dk in known_keys}
         for package in packages:
-            sig_date = package.signature.creation_time.replace(tzinfo=pytz.utc)
+            bad = False
+            sig = package.signature
+            sig_date = sig.creation_time.replace(tzinfo=pytz.utc)
             package.sig_date = sig_date.date()
-            key_id = package.signature.key_id
-            signer = finder.find_by_pgp_key(key_id)
-            package.sig_by = signer or key_id
-            if signer is None or signer.id != package.packager_id:
-                filtered.append(package)
-            elif sig_date > package.build_date + cutoff:
+            dev_key = known_keys.get(sig.key_id, None)
+            if dev_key:
+                package.sig_by = dev_key.owner
+                if dev_key.owner_id != package.packager_id:
+                    bad = True
+            else:
+                package.sig_by = sig.key_id
+                bad = True
+
+            if sig_date > package.build_date + cutoff:
+                bad = True
+
+            if bad:
                 filtered.append(package)
         packages = filtered
     else:
-- 
cgit v1.2.3-2-g168b


From f98ff8cd22185c11dccdbe19b5bb7ed849b38e6b Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Mon, 4 Feb 2013 20:02:00 -0600
Subject: Add './' hack to generate_keyring as well

If you specify a relative path to gpg without a slash character, it
interprets as relative to ~/.gnupg, which is stupid.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/generate_keyring.py | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'devel')

diff --git a/devel/management/commands/generate_keyring.py b/devel/management/commands/generate_keyring.py
index 15ae488d..34bcd2f8 100644
--- a/devel/management/commands/generate_keyring.py
+++ b/devel/management/commands/generate_keyring.py
@@ -55,6 +55,10 @@ def generate_keyring(keyserver, keyring):
     master_key_ids = MasterKey.objects.values_list("pgp_key", flat=True)
     logger.info("%d keys fetched from master keys", len(master_key_ids))
 
+    # GPG is stupid and interprets any filename without path portion as being
+    # in ~/.gnupg/. Fake it out if we just get a bare filename.
+    if '/' not in keyring:
+        keyring = './%s' % keyring
     gpg_cmd = ["gpg", "--no-default-keyring", "--keyring", keyring,
             "--keyserver", keyserver, "--recv-keys"]
     logger.info("running command: %r", gpg_cmd)
-- 
cgit v1.2.3-2-g168b


From 55179a4f9e8b80d515bae7032af8aefc33ae0192 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Wed, 16 Jan 2013 16:07:26 -0600
Subject: reporead: remove batched_bulk_create

Now that Django 1.5 is out and realized SQLite3 only allows for 999
parameters per SQL call, we don't need to manually batch things up
anymore and can let the underlying bulk_create code do it for us.

This basically reverts commit 88ee61a39ac3.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/management/commands/reporead.py | 32 +++++---------------------------
 1 file changed, 5 insertions(+), 27 deletions(-)

(limited to 'devel')

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index ab0efeed..ccac55f2 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -32,7 +32,6 @@ from django.utils.timezone import now
 
 from devel.utils import UserFinder
 from main.models import Arch, Package, PackageFile, Repo
-from main.utils import database_vendor
 from packages.models import Depend, Conflict, Provision, Replacement, Update
 from packages.utils import parse_version
 
@@ -182,27 +181,6 @@ def create_related(model, package, rel_str, equals_only=False):
     return related
 
 
-def batched_bulk_create(model, all_objects):
-    # for short lists, just bulk_create as we should be fine
-    if len(all_objects) < 20:
-        return model.objects.bulk_create(all_objects)
-
-    if database_vendor(model, mode='write') == 'sqlite':
-        # 999 max variables in each SQL statement
-        incr = 999 // len(model._meta.fields)
-    else:
-        incr = 1000
-
-    def chunks():
-        offset = 0
-        while offset < len(all_objects):
-            yield all_objects[offset:offset + incr]
-            offset += incr
-
-    for items in chunks():
-        model.objects.bulk_create(items)
-
-
 def create_multivalued(dbpkg, repopkg, db_attr, repo_attr):
     '''Populate the simplest of multivalued attributes. These are those that
     only deal with a 'name' attribute, such as licenses, groups, etc. The input
@@ -256,20 +234,20 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None):
     deps += [create_depend(dbpkg, y, 'O') for y in repopkg.optdepends]
     deps += [create_depend(dbpkg, y, 'M') for y in repopkg.makedepends]
     deps += [create_depend(dbpkg, y, 'C') for y in repopkg.checkdepends]
-    batched_bulk_create(Depend, deps)
+    Depend.objects.bulk_create(deps)
 
     dbpkg.conflicts.all().delete()
     conflicts = [create_related(Conflict, dbpkg, y) for y in repopkg.conflicts]
-    batched_bulk_create(Conflict, conflicts)
+    Conflict.objects.bulk_create(conflicts)
 
     dbpkg.provides.all().delete()
     provides = [create_related(Provision, dbpkg, y, equals_only=True)
             for y in repopkg.provides]
-    batched_bulk_create(Provision, provides)
+    Provision.objects.bulk_create(provides)
 
     dbpkg.replaces.all().delete()
     replaces = [create_related(Replacement, dbpkg, y) for y in repopkg.replaces]
-    batched_bulk_create(Replacement, replaces)
+    Replacement.objects.bulk_create(replaces)
 
     create_multivalued(dbpkg, repopkg, 'groups', 'groups')
     create_multivalued(dbpkg, repopkg, 'licenses', 'license')
@@ -319,7 +297,7 @@ def populate_files(dbpkg, repopkg, force=False):
                     directory=dirname,
                     filename=filename)
             pkg_files.append(pkgfile)
-        batched_bulk_create(PackageFile, pkg_files)
+        PackageFile.objects.bulk_create(pkg_files)
         dbpkg.files_last_update = now()
         dbpkg.save()
 
-- 
cgit v1.2.3-2-g168b


From dd0ecfaeaceb1e1b8a185800de35f0f6e741feac Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Tue, 26 Feb 2013 19:51:40 -0600
Subject: Use user.userprofile rather than user.get_profile()

The get_profile() function is deprecated as of Django 1.5.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 devel/views.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'devel')

diff --git a/devel/views.py b/devel/views.py
index ff1dec12..61c1e568 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -152,7 +152,7 @@ def change_profile(request):
     if request.POST:
         form = ProfileForm(request.POST)
         profile_form = UserProfileForm(request.POST, request.FILES,
-                instance=request.user.get_profile())
+                instance=request.user.userprofile)
         if form.is_valid() and profile_form.is_valid():
             request.user.email = form.cleaned_data['email']
             if form.cleaned_data['passwd1']:
@@ -163,7 +163,7 @@ def change_profile(request):
             return HttpResponseRedirect('/devel/')
     else:
         form = ProfileForm(initial={'email': request.user.email})
-        profile_form = UserProfileForm(instance=request.user.get_profile())
+        profile_form = UserProfileForm(instance=request.user.userprofile)
     return render(request, 'devel/profile.html',
             {'form': form, 'profile_form': profile_form})
 
-- 
cgit v1.2.3-2-g168b