Merge branch 'master' of git://projects.archlinux.org/archweb

Conflicts:
	packages/urls.py
	templates/packages/details.html

5 files changed, 255 insertions, 53 deletions

diff --git a/devel/management/commands/rematch_packager.py b/devel/management/commands/rematch_packager.py
new file mode 100644
index 00000000..ba6e6a54
--- /dev/null
+++ b/devel/management/commands/rematch_packager.py
@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+"""
+rematch_packager command
+
+Match all packages with a packager_str but NULL packager_id to a packager if we
+can find one.
+
+Usage: ./manage.py rematch_packager
+"""
+
+from django.core.management.base import NoArgsCommand
+
+import sys
+import logging
+
+from devel.utils import UserFinder
+from main.models import Package
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s -> %(levelname)s: %(message)s',
+    datefmt='%Y-%m-%d %H:%M:%S',
+    stream=sys.stderr)
+logger = logging.getLogger()
+
+class Command(NoArgsCommand):
+    help = "Runs a check on all active mirror URLs to determine if they are reachable via IPv4 and/or v6."
+
+    def handle_noargs(self, **options):
+        v = int(options.get('verbosity', None))
+        if v == 0:
+            logger.level = logging.ERROR
+        elif v == 1:
+            logger.level = logging.INFO
+        elif v == 2:
+            logger.level = logging.DEBUG
+
+        return match_packager()
+
+def match_packager():
+    finder = UserFinder()
+    logger.info("getting all unmatched packages")
+    package_count = matched_count = 0
+    unknown = set()
+
+    for package in Package.objects.filter(packager__isnull=True):
+        logger.debug("package %s, packager string %s",
+                package.pkgname, package.packager_str)
+        package_count += 1
+        user = finder.find(package.packager_str)
+        if user:
+            package.packager = user
+            logger.debug("  found user %s" % user.username)
+            package.save()
+            matched_count += 1
+        else:
+            unknown.add(package.packager_str)
+
+    logger.info("%d packages checked, %d newly matched",
+            package_count, matched_count)
+    logger.debug("unknown packagers:\n%s",
+            "\n".join(unknown))
+
+# vim: set ts=4 sw=4 et:
diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index baf7fee1..470b785d 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -16,7 +16,6 @@ Example:
 from django.core.management.base import BaseCommand, CommandError
 from django.contrib.auth.models import User
 from django.db import transaction
-from django.db.models import Q
 
 from collections import defaultdict
 import io
@@ -28,6 +27,7 @@ import logging
 from datetime import datetime
 from optparse import make_option
 
+from devel.utils import UserFinder
 from main.models import Arch, Package, PackageDepend, PackageFile, Repo
 from packages.models import Conflict, Provision, Replacement
 
@@ -130,55 +130,6 @@ class Pkg(object):
         return u'%s-%s' % (self.ver, self.rel)
 
 
-def find_user(userstring):
-    '''
-    Attempt to find the corresponding User object for a standard
-    packager string, e.g. something like
-        'A. U. Thor <author@example.com>'.
-    We start by searching for a matching email address; we then move onto
-    matching by first/last name. If we cannot find a user, then return None.
-    '''
-    if userstring in find_user.cache:
-        return find_user.cache[userstring]
-    matches = re.match(r'^([^<]+)? ?<([^>]*)>', userstring)
-    if not matches:
-        return None
-
-    user = None
-    name = matches.group(1)
-    email = matches.group(2)
-
-    def user_email():
-        return User.objects.get(email=email)
-    def profile_email():
-        return User.objects.get(userprofile__public_email=email)
-    def user_name():
-        # yes, a bit odd but this is the easiest way since we can't always be
-        # sure how to split the name. Ensure every 'token' appears in at least
-        # one of the two name fields.
-        name_q = Q()
-        for token in name.split():
-            # ignore quoted parts; e.g. nicknames in strings
-            if re.match(r'^[\'"].*[\'"]$', token):
-                continue
-            name_q &= (Q(first_name__icontains=token) |
-                    Q(last_name__icontains=token))
-        return User.objects.get(name_q)
-
-    for matcher in (user_email, profile_email, user_name):
-        try:
-            user = matcher()
-            break
-        except (User.DoesNotExist, User.MultipleObjectsReturned):
-            pass
-
-    find_user.cache[userstring] = user
-    return user
-
-# cached mappings of user strings -> User objects so we don't have to do the
-# lookup more than strictly necessary.
-find_user.cache = {}
-
 DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.*))?$")
 
 def create_depend(package, dep_str, optional=False):
@@ -231,6 +182,8 @@ def create_multivalued(dbpkg, repopkg, db_attr, repo_attr):
     for name in getattr(repopkg, repo_attr):
         collection.create(name=name)
 
+finder = UserFinder()
+
 def populate_pkg(dbpkg, repopkg, force=False, timestamp=None):
     db_score = 1
 
@@ -249,7 +202,7 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None):
     dbpkg.build_date = repopkg.builddate
     dbpkg.packager_str = repopkg.packager
     # attempt to find the corresponding django user for this string
-    dbpkg.packager = find_user(repopkg.packager)
+    dbpkg.packager = finder.find(repopkg.packager)
 
     if timestamp:
         dbpkg.flag_date = None
diff --git a/devel/tests.py b/devel/tests.py
index 604da74c..36691179 100644
--- a/devel/tests.py
+++ b/devel/tests.py
@@ -1,8 +1,10 @@
 from django.test import TestCase
 
+from django.contrib.auth.models import User
+from devel.utils import UserFinder
+from main.models import UserProfile
 
 class DevelTest(TestCase):
-
     def test_index(self):
         response = self.client.get('/devel/')
         self.assertEqual(response.status_code, 302)
@@ -27,3 +29,87 @@ class DevelTest(TestCase):
     def test_mirrors(self):
         response = self.client.get('/mirrors/')
         self.assertEqual(response.status_code, 200)
+
+class FindUserTest(TestCase):
+
+    def setUp(self):
+        self.finder = UserFinder()
+
+        self.user1 = User.objects.create(username="joeuser", first_name="Joe",
+                last_name="User", email="user1@example.com")
+        self.user2 = User.objects.create(username="john", first_name="John",
+                last_name="", email="user2@example.com")
+        self.user3 = User.objects.create(username="bjones", first_name="Bob",
+                last_name="Jones", email="user3@example.com")
+
+        for user in (self.user1, self.user2, self.user3):
+            email_addr = "%s@awesome.com" % user.username
+            UserProfile.objects.create(user=user, public_email=email_addr)
+
+        self.user4 = User.objects.create(username="tim1", first_name="Tim",
+                last_name="One", email="tim@example.com")
+        self.user5 = User.objects.create(username="tim2", first_name="Tim",
+                last_name="Two", email="timtwo@example.com")
+
+    def test_not_matching(self):
+        self.assertIsNone(self.finder.find(None))
+        self.assertIsNone(self.finder.find(""))
+        self.assertIsNone(self.finder.find("Bogus"))
+        self.assertIsNone(self.finder.find("Bogus <invalid"))
+        self.assertIsNone(self.finder.find("Bogus User <bogus@example.com>"))
+        self.assertIsNone(self.finder.find("<bogus@example.com>"))
+        self.assertIsNone(self.finder.find("bogus@example.com"))
+        self.assertIsNone(self.finder.find("Unknown Packager"))
+
+    def test_by_email(self):
+        self.assertEqual(self.user1,
+                self.finder.find("XXX YYY <user1@example.com>"))
+        self.assertEqual(self.user2,
+                self.finder.find("YYY ZZZ <user2@example.com>"))
+
+    def test_by_profile_email(self):
+        self.assertEqual(self.user1,
+                self.finder.find("XXX <joeuser@awesome.com>"))
+        self.assertEqual(self.user2,
+                self.finder.find("YYY <john@awesome.com>"))
+        self.assertEqual(self.user3,
+                self.finder.find("ZZZ <bjones@awesome.com>"))
+
+    def test_by_name(self):
+        self.assertEqual(self.user1,
+                self.finder.find("Joe User <joe@differentdomain.com>"))
+        self.assertEqual(self.user1,
+                self.finder.find("Joe User"))
+        self.assertEqual(self.user2,
+                self.finder.find("John <john@differentdomain.com>"))
+        self.assertEqual(self.user2,
+                self.finder.find("John"))
+        self.assertEqual(self.user3,
+                self.finder.find("Bob Jones <bjones AT Arch Linux DOT org>"))
+
+    def test_by_invalid(self):
+        self.assertEqual(self.user1,
+                self.finder.find("Joe User <user1@example.com"))
+        self.assertEqual(self.user1,
+                self.finder.find("Joe 'nickname' User <user1@example.com"))
+        self.assertEqual(self.user1,
+                self.finder.find("Joe \"nickname\" User <user1@example.com"))
+        self.assertEqual(self.user1,
+                self.finder.find("Joe User <joe@differentdomain.com"))
+
+    def test_cache(self):
+        # simply look two of them up, but then do it repeatedly
+        for i in range(50):
+            self.assertEqual(self.user1,
+                    self.finder.find("XXX YYY <user1@example.com>"))
+            self.assertEqual(self.user3,
+                    self.finder.find("Bob Jones <bjones AT Arch Linux DOT org>"))
+
+    def test_ambiguous(self):
+        self.assertEqual(self.user4,
+                self.finder.find("Tim One <tim@anotherdomain.com>"))
+        self.assertEqual(self.user5,
+                self.finder.find("Tim Two <tim@anotherdomain.com>"))
+        self.assertIsNone(self.finder.find("Tim <tim@anotherdomain.com>"))
+
+# vim: set ts=4 sw=4 et:
diff --git a/devel/utils.py b/devel/utils.py
index abfdabe5..d7a154a8 100644
--- a/devel/utils.py
+++ b/devel/utils.py
@@ -1,7 +1,11 @@
+import re
+
 from django.contrib.auth.models import User
 from django.db import connection
+from django.db.models import Count, Q
 
 from main.utils import cache_function
+from main.models import Package
 from packages.models import PackageRelation
 
 @cache_function(300)
@@ -28,10 +32,86 @@ SELECT pr.user_id, COUNT(*), COUNT(p.flag_date)
         pkg_count[k] = total
         flag_count[k] = flagged
 
+    update_count = Package.objects.values_list('packager').order_by(
+            'packager').annotate(Count('packager'))
+    update_count = dict(update_count)
+
     for m in maintainers:
         m.package_count = pkg_count.get(m.id, 0)
         m.flagged_count = flag_count.get(m.id, 0)
+        m.updated_count = update_count.get(m.id, 0)
 
     return maintainers
 
+
+class UserFinder(object):
+    def __init__(self):
+        self.cache = {}
+
+    @staticmethod
+    def user_email(name, email):
+        if email:
+            return User.objects.get(email=email)
+        return None
+
+    @staticmethod
+    def profile_email(name, email):
+        if email:
+            return User.objects.get(userprofile__public_email=email)
+        return None
+
+    @staticmethod
+    def user_name(name, email):
+        # yes, a bit odd but this is the easiest way since we can't always be
+        # sure how to split the name. Ensure every 'token' appears in at least
+        # one of the two name fields.
+        if not name:
+            return None
+        name_q = Q()
+        for token in name.split():
+            # ignore quoted parts; e.g. nicknames in strings
+            if re.match(r'^[\'"].*[\'"]$', token):
+                continue
+            name_q &= (Q(first_name__icontains=token) |
+                    Q(last_name__icontains=token))
+        return User.objects.get(name_q)
+
+    def find(self, userstring):
+        '''
+        Attempt to find the corresponding User object for a standard
+        packager string, e.g. something like
+            'A. U. Thor <author@example.com>'.
+        We start by searching for a matching email address; we then move onto
+        matching by first/last name. If we cannot find a user, then return None.
+        '''
+        if not userstring:
+            return None
+        if userstring in self.cache:
+            return self.cache[userstring]
+
+        name = email = None
+
+        matches = re.match(r'^([^<]+)? ?<([^>]*)>?', userstring)
+        if not matches:
+            name = userstring.strip()
+        else:
+            name = matches.group(1)
+            email = matches.group(2)
+
+        user = None
+        find_methods = (self.user_email, self.profile_email, self.user_name)
+        for matcher in find_methods:
+            try:
+                user = matcher(name, email)
+                if user != None:
+                    break
+            except (User.DoesNotExist, User.MultipleObjectsReturned):
+                pass
+
+        self.cache[userstring] = user
+        return user
+
+    def clear_cache(self):
+        self.cache = {}
+
 # vim: set ts=4 sw=4 et:
diff --git a/devel/views.py b/devel/views.py
index 3bf68a58..27c32e7b 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -6,7 +6,7 @@ from django.contrib.auth.models import User, Group
 from django.contrib.sites.models import Site
 from django.core.mail import send_mail
 from django.db import transaction
-from django.db.models import Q
+from django.db.models import F, Q
 from django.http import Http404
 from django.shortcuts import get_object_or_404
 from django.template import loader, Context
@@ -50,9 +50,11 @@ def index(request):
     total_orphans = Package.objects.exclude(pkgbase__in=maintained).count()
     total_flagged_orphans = Package.objects.filter(
             flag_date__isnull=False).exclude(pkgbase__in=maintained).count()
+    total_updated = Package.objects.filter(packager__isnull=True).count()
     orphan = {
             'package_count': total_orphans,
             'flagged_count': total_flagged_orphans,
+            'updated_count': total_updated,
     }
 
     page_dict = {
@@ -161,6 +163,23 @@ def report(request, report, username=None):
                 package.compressed_size)
             package.installed_size_pretty = filesizeformat(
                 package.installed_size)
+    elif report == 'badcompression':
+        title = 'Packages that have little need for compression'
+        cutoff = 0.90 * F('installed_size')
+        packages = packages.filter(compressed_size__gt=0, installed_size__gt=0,
+                compressed_size__gte=cutoff).order_by('-compressed_size')
+        names = [ 'Compressed Size', 'Installed Size', 'Ratio', 'Type' ]
+        attrs = [ 'compressed_size_pretty', 'installed_size_pretty',
+                'ratio', 'compress_type' ]
+        # Format the compressed and installed sizes with MB/GB/etc suffixes
+        for package in packages:
+            package.compressed_size_pretty = filesizeformat(
+                package.compressed_size)
+            package.installed_size_pretty = filesizeformat(
+                package.installed_size)
+            ratio = package.compressed_size / float(package.installed_size)
+            package.ratio = '%.2f' % ratio
+            package.compress_type = package.filename.split('.')[-1]
     elif report == 'uncompressed-man':
         title = 'Packages with uncompressed manpages'
         # magic going on here! Checking for all '.1'...'.9' extensions