summaryrefslogtreecommitdiff
path: root/devel
diff options
context:
space:
mode:
Diffstat (limited to 'devel')
-rw-r--r--devel/management/commands/rematch_packager.py64
-rw-r--r--devel/management/commands/reporead.py55
-rw-r--r--devel/tests.py88
-rw-r--r--devel/utils.py80
-rw-r--r--devel/views.py21
5 files changed, 255 insertions, 53 deletions
diff --git a/devel/management/commands/rematch_packager.py b/devel/management/commands/rematch_packager.py
new file mode 100644
index 00000000..ba6e6a54
--- /dev/null
+++ b/devel/management/commands/rematch_packager.py
@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+"""
+rematch_packager command
+
+Match all packages with a packager_str but NULL packager_id to a packager if we
+can find one.
+
+Usage: ./manage.py rematch_packager
+"""
+
+from django.core.management.base import NoArgsCommand
+
+import sys
+import logging
+
+from devel.utils import UserFinder
+from main.models import Package
+
+logging.basicConfig(
+ level=logging.INFO,
+ format='%(asctime)s -> %(levelname)s: %(message)s',
+ datefmt='%Y-%m-%d %H:%M:%S',
+ stream=sys.stderr)
+logger = logging.getLogger()
+
+class Command(NoArgsCommand):
+ help = "Runs a check on all active mirror URLs to determine if they are reachable via IPv4 and/or v6."
+
+ def handle_noargs(self, **options):
+ v = int(options.get('verbosity', None))
+ if v == 0:
+ logger.level = logging.ERROR
+ elif v == 1:
+ logger.level = logging.INFO
+ elif v == 2:
+ logger.level = logging.DEBUG
+
+ return match_packager()
+
+def match_packager():
+ finder = UserFinder()
+ logger.info("getting all unmatched packages")
+ package_count = matched_count = 0
+ unknown = set()
+
+ for package in Package.objects.filter(packager__isnull=True):
+ logger.debug("package %s, packager string %s",
+ package.pkgname, package.packager_str)
+ package_count += 1
+ user = finder.find(package.packager_str)
+ if user:
+ package.packager = user
+ logger.debug(" found user %s" % user.username)
+ package.save()
+ matched_count += 1
+ else:
+ unknown.add(package.packager_str)
+
+ logger.info("%d packages checked, %d newly matched",
+ package_count, matched_count)
+ logger.debug("unknown packagers:\n%s",
+ "\n".join(unknown))
+
+# vim: set ts=4 sw=4 et:
diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index baf7fee1..470b785d 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -16,7 +16,6 @@ Example:
from django.core.management.base import BaseCommand, CommandError
from django.contrib.auth.models import User
from django.db import transaction
-from django.db.models import Q
from collections import defaultdict
import io
@@ -28,6 +27,7 @@ import logging
from datetime import datetime
from optparse import make_option
+from devel.utils import UserFinder
from main.models import Arch, Package, PackageDepend, PackageFile, Repo
from packages.models import Conflict, Provision, Replacement
@@ -130,55 +130,6 @@ class Pkg(object):
return u'%s-%s' % (self.ver, self.rel)
-def find_user(userstring):
- '''
- Attempt to find the corresponding User object for a standard
- packager string, e.g. something like
- 'A. U. Thor <author@example.com>'.
- We start by searching for a matching email address; we then move onto
- matching by first/last name. If we cannot find a user, then return None.
- '''
- if userstring in find_user.cache:
- return find_user.cache[userstring]
- matches = re.match(r'^([^<]+)? ?<([^>]*)>', userstring)
- if not matches:
- return None
-
- user = None
- name = matches.group(1)
- email = matches.group(2)
-
- def user_email():
- return User.objects.get(email=email)
- def profile_email():
- return User.objects.get(userprofile__public_email=email)
- def user_name():
- # yes, a bit odd but this is the easiest way since we can't always be
- # sure how to split the name. Ensure every 'token' appears in at least
- # one of the two name fields.
- name_q = Q()
- for token in name.split():
- # ignore quoted parts; e.g. nicknames in strings
- if re.match(r'^[\'"].*[\'"]$', token):
- continue
- name_q &= (Q(first_name__icontains=token) |
- Q(last_name__icontains=token))
- return User.objects.get(name_q)
-
- for matcher in (user_email, profile_email, user_name):
- try:
- user = matcher()
- break
- except (User.DoesNotExist, User.MultipleObjectsReturned):
- pass
-
- find_user.cache[userstring] = user
- return user
-
-# cached mappings of user strings -> User objects so we don't have to do the
-# lookup more than strictly necessary.
-find_user.cache = {}
-
DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.*))?$")
def create_depend(package, dep_str, optional=False):
@@ -231,6 +182,8 @@ def create_multivalued(dbpkg, repopkg, db_attr, repo_attr):
for name in getattr(repopkg, repo_attr):
collection.create(name=name)
+finder = UserFinder()
+
def populate_pkg(dbpkg, repopkg, force=False, timestamp=None):
db_score = 1
@@ -249,7 +202,7 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None):
dbpkg.build_date = repopkg.builddate
dbpkg.packager_str = repopkg.packager
# attempt to find the corresponding django user for this string
- dbpkg.packager = find_user(repopkg.packager)
+ dbpkg.packager = finder.find(repopkg.packager)
if timestamp:
dbpkg.flag_date = None
diff --git a/devel/tests.py b/devel/tests.py
index 604da74c..36691179 100644
--- a/devel/tests.py
+++ b/devel/tests.py
@@ -1,8 +1,10 @@
from django.test import TestCase
+from django.contrib.auth.models import User
+from devel.utils import UserFinder
+from main.models import UserProfile
class DevelTest(TestCase):
-
def test_index(self):
response = self.client.get('/devel/')
self.assertEqual(response.status_code, 302)
@@ -27,3 +29,87 @@ class DevelTest(TestCase):
def test_mirrors(self):
response = self.client.get('/mirrors/')
self.assertEqual(response.status_code, 200)
+
+class FindUserTest(TestCase):
+
+ def setUp(self):
+ self.finder = UserFinder()
+
+ self.user1 = User.objects.create(username="joeuser", first_name="Joe",
+ last_name="User", email="user1@example.com")
+ self.user2 = User.objects.create(username="john", first_name="John",
+ last_name="", email="user2@example.com")
+ self.user3 = User.objects.create(username="bjones", first_name="Bob",
+ last_name="Jones", email="user3@example.com")
+
+ for user in (self.user1, self.user2, self.user3):
+ email_addr = "%s@awesome.com" % user.username
+ UserProfile.objects.create(user=user, public_email=email_addr)
+
+ self.user4 = User.objects.create(username="tim1", first_name="Tim",
+ last_name="One", email="tim@example.com")
+ self.user5 = User.objects.create(username="tim2", first_name="Tim",
+ last_name="Two", email="timtwo@example.com")
+
+ def test_not_matching(self):
+ self.assertIsNone(self.finder.find(None))
+ self.assertIsNone(self.finder.find(""))
+ self.assertIsNone(self.finder.find("Bogus"))
+ self.assertIsNone(self.finder.find("Bogus <invalid"))
+ self.assertIsNone(self.finder.find("Bogus User <bogus@example.com>"))
+ self.assertIsNone(self.finder.find("<bogus@example.com>"))
+ self.assertIsNone(self.finder.find("bogus@example.com"))
+ self.assertIsNone(self.finder.find("Unknown Packager"))
+
+ def test_by_email(self):
+ self.assertEqual(self.user1,
+ self.finder.find("XXX YYY <user1@example.com>"))
+ self.assertEqual(self.user2,
+ self.finder.find("YYY ZZZ <user2@example.com>"))
+
+ def test_by_profile_email(self):
+ self.assertEqual(self.user1,
+ self.finder.find("XXX <joeuser@awesome.com>"))
+ self.assertEqual(self.user2,
+ self.finder.find("YYY <john@awesome.com>"))
+ self.assertEqual(self.user3,
+ self.finder.find("ZZZ <bjones@awesome.com>"))
+
+ def test_by_name(self):
+ self.assertEqual(self.user1,
+ self.finder.find("Joe User <joe@differentdomain.com>"))
+ self.assertEqual(self.user1,
+ self.finder.find("Joe User"))
+ self.assertEqual(self.user2,
+ self.finder.find("John <john@differentdomain.com>"))
+ self.assertEqual(self.user2,
+ self.finder.find("John"))
+ self.assertEqual(self.user3,
+ self.finder.find("Bob Jones <bjones AT Arch Linux DOT org>"))
+
+ def test_by_invalid(self):
+ self.assertEqual(self.user1,
+ self.finder.find("Joe User <user1@example.com"))
+ self.assertEqual(self.user1,
+ self.finder.find("Joe 'nickname' User <user1@example.com"))
+ self.assertEqual(self.user1,
+ self.finder.find("Joe \"nickname\" User <user1@example.com"))
+ self.assertEqual(self.user1,
+ self.finder.find("Joe User <joe@differentdomain.com"))
+
+ def test_cache(self):
+ # simply look two of them up, but then do it repeatedly
+ for i in range(50):
+ self.assertEqual(self.user1,
+ self.finder.find("XXX YYY <user1@example.com>"))
+ self.assertEqual(self.user3,
+ self.finder.find("Bob Jones <bjones AT Arch Linux DOT org>"))
+
+ def test_ambiguous(self):
+ self.assertEqual(self.user4,
+ self.finder.find("Tim One <tim@anotherdomain.com>"))
+ self.assertEqual(self.user5,
+ self.finder.find("Tim Two <tim@anotherdomain.com>"))
+ self.assertIsNone(self.finder.find("Tim <tim@anotherdomain.com>"))
+
+# vim: set ts=4 sw=4 et:
diff --git a/devel/utils.py b/devel/utils.py
index abfdabe5..d7a154a8 100644
--- a/devel/utils.py
+++ b/devel/utils.py
@@ -1,7 +1,11 @@
+import re
+
from django.contrib.auth.models import User
from django.db import connection
+from django.db.models import Count, Q
from main.utils import cache_function
+from main.models import Package
from packages.models import PackageRelation
@cache_function(300)
@@ -28,10 +32,86 @@ SELECT pr.user_id, COUNT(*), COUNT(p.flag_date)
pkg_count[k] = total
flag_count[k] = flagged
+ update_count = Package.objects.values_list('packager').order_by(
+ 'packager').annotate(Count('packager'))
+ update_count = dict(update_count)
+
for m in maintainers:
m.package_count = pkg_count.get(m.id, 0)
m.flagged_count = flag_count.get(m.id, 0)
+ m.updated_count = update_count.get(m.id, 0)
return maintainers
+
+class UserFinder(object):
+ def __init__(self):
+ self.cache = {}
+
+ @staticmethod
+ def user_email(name, email):
+ if email:
+ return User.objects.get(email=email)
+ return None
+
+ @staticmethod
+ def profile_email(name, email):
+ if email:
+ return User.objects.get(userprofile__public_email=email)
+ return None
+
+ @staticmethod
+ def user_name(name, email):
+ # yes, a bit odd but this is the easiest way since we can't always be
+ # sure how to split the name. Ensure every 'token' appears in at least
+ # one of the two name fields.
+ if not name:
+ return None
+ name_q = Q()
+ for token in name.split():
+ # ignore quoted parts; e.g. nicknames in strings
+ if re.match(r'^[\'"].*[\'"]$', token):
+ continue
+ name_q &= (Q(first_name__icontains=token) |
+ Q(last_name__icontains=token))
+ return User.objects.get(name_q)
+
+ def find(self, userstring):
+ '''
+ Attempt to find the corresponding User object for a standard
+ packager string, e.g. something like
+ 'A. U. Thor <author@example.com>'.
+ We start by searching for a matching email address; we then move onto
+ matching by first/last name. If we cannot find a user, then return None.
+ '''
+ if not userstring:
+ return None
+ if userstring in self.cache:
+ return self.cache[userstring]
+
+ name = email = None
+
+ matches = re.match(r'^([^<]+)? ?<([^>]*)>?', userstring)
+ if not matches:
+ name = userstring.strip()
+ else:
+ name = matches.group(1)
+ email = matches.group(2)
+
+ user = None
+ find_methods = (self.user_email, self.profile_email, self.user_name)
+ for matcher in find_methods:
+ try:
+ user = matcher(name, email)
+ if user != None:
+ break
+ except (User.DoesNotExist, User.MultipleObjectsReturned):
+ pass
+
+ self.cache[userstring] = user
+ return user
+
+ def clear_cache(self):
+ self.cache = {}
+
# vim: set ts=4 sw=4 et:
diff --git a/devel/views.py b/devel/views.py
index 3bf68a58..27c32e7b 100644
--- a/devel/views.py
+++ b/devel/views.py
@@ -6,7 +6,7 @@ from django.contrib.auth.models import User, Group
from django.contrib.sites.models import Site
from django.core.mail import send_mail
from django.db import transaction
-from django.db.models import Q
+from django.db.models import F, Q
from django.http import Http404
from django.shortcuts import get_object_or_404
from django.template import loader, Context
@@ -50,9 +50,11 @@ def index(request):
total_orphans = Package.objects.exclude(pkgbase__in=maintained).count()
total_flagged_orphans = Package.objects.filter(
flag_date__isnull=False).exclude(pkgbase__in=maintained).count()
+ total_updated = Package.objects.filter(packager__isnull=True).count()
orphan = {
'package_count': total_orphans,
'flagged_count': total_flagged_orphans,
+ 'updated_count': total_updated,
}
page_dict = {
@@ -161,6 +163,23 @@ def report(request, report, username=None):
package.compressed_size)
package.installed_size_pretty = filesizeformat(
package.installed_size)
+ elif report == 'badcompression':
+ title = 'Packages that have little need for compression'
+ cutoff = 0.90 * F('installed_size')
+ packages = packages.filter(compressed_size__gt=0, installed_size__gt=0,
+ compressed_size__gte=cutoff).order_by('-compressed_size')
+ names = [ 'Compressed Size', 'Installed Size', 'Ratio', 'Type' ]
+ attrs = [ 'compressed_size_pretty', 'installed_size_pretty',
+ 'ratio', 'compress_type' ]
+ # Format the compressed and installed sizes with MB/GB/etc suffixes
+ for package in packages:
+ package.compressed_size_pretty = filesizeformat(
+ package.compressed_size)
+ package.installed_size_pretty = filesizeformat(
+ package.installed_size)
+ ratio = package.compressed_size / float(package.installed_size)
+ package.ratio = '%.2f' % ratio
+ package.compress_type = package.filename.split('.')[-1]
elif report == 'uncompressed-man':
title = 'Packages with uncompressed manpages'
# magic going on here! Checking for all '.1'...'.9' extensions
generated by cgit v1.2.3-2-g168b (git 2.25.1) at 2024-11-02 23:14:46 +0000