From d80908e21e88c78262563f5852d81f2754fb5d82 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 23 Dec 2013 09:44:08 -0600 Subject: Change import location of django_countries fields This will work with both the newer and older versions. Signed-off-by: Dan McGee --- devel/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'devel') diff --git a/devel/models.py b/devel/models.py index 44bbc66e..5c4d4fe7 100644 --- a/devel/models.py +++ b/devel/models.py @@ -4,7 +4,7 @@ import pytz from django.db import models from django.db.models.signals import pre_save from django.contrib.auth.models import User -from django_countries import CountryField +from django_countries.fields import CountryField from .fields import PGPKeyField from main.utils import make_choice, set_created_field -- cgit v1.2.3-2-g168b From 4a5714fba769c83724f29578d573f0b4096eb332 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 26 Jan 2014 12:55:33 -0600 Subject: Revert "Change old packages report from two years to one year" This reverts commit 8d3a1a1c504a70dd23d36c3ed5be0ebcd2f7a86d. Turns out we aren't updating packages quite as often anymore. There are currently 1900+ packages in the repos built more than one year ago. Signed-off-by: Dan McGee Conflicts: devel/views.py --- devel/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'devel') diff --git a/devel/views.py b/devel/views.py index 29954b51..46feece2 100644 --- a/devel/views.py +++ b/devel/views.py @@ -194,8 +194,8 @@ def report(request, report_name, username=None): type=PackageRelation.MAINTAINER).values('user')) if report_name == 'old': - title = 'Packages last built more than one year ago' - cutoff = now() - timedelta(days=365) + title = 'Packages last built more than two years ago' + cutoff = now() - timedelta(days=365 * 2) packages = packages.filter( build_date__lt=cutoff).order_by('build_date') elif report_name == 'long-out-of-date': -- cgit v1.2.3-2-g168b From 5fcc6af4ddd0d6f9fbe491a1fd5a9a0edc5b52a5 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 22 Feb 2014 08:42:07 -0600 Subject: Change long out-of-date report to 30 days At the request of barthalion. This bumps the report from currently showing 36 packages to 109 packages. Signed-off-by: Dan McGee --- devel/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'devel') diff --git a/devel/views.py b/devel/views.py index 46feece2..1e20a437 100644 --- a/devel/views.py +++ b/devel/views.py @@ -199,8 +199,8 @@ def report(request, report_name, username=None): packages = packages.filter( build_date__lt=cutoff).order_by('build_date') elif report_name == 'long-out-of-date': - title = 'Packages marked out-of-date more than 90 days ago' - cutoff = now() - timedelta(days=90) + title = 'Packages marked out-of-date more than 30 days ago' + cutoff = now() - timedelta(days=30) packages = packages.filter( flag_date__lt=cutoff).order_by('flag_date') elif report_name == 'big': -- cgit v1.2.3-2-g168b From 7947d36c4e509a917941a34c576fde3a207a4439 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 22 Feb 2014 13:38:59 -0600 Subject: Break out developer reports into a separate module This code was getting quite unwieldy, and wasn't very modular. Introduce a DeveloperReport class that contains the content for a single report, and utilize it to create our various report metadata and package filtering operations. Utilize these report objects in the reports view, vastly simplifying it. We don't yet dynamically generate the list of reports on the developer index page; that will be coming soon. Signed-off-by: Dan McGee --- devel/reports.py | 176 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ devel/views.py | 137 +++++-------------------------------------- 2 files changed, 191 insertions(+), 122 deletions(-) create mode 100644 devel/reports.py (limited to 'devel') diff --git a/devel/reports.py b/devel/reports.py new file mode 100644 index 00000000..f1ffab3f --- /dev/null +++ b/devel/reports.py @@ -0,0 +1,176 @@ +from datetime import timedelta +import pytz + +from django.db.models import F +from django.template.defaultfilters import filesizeformat +from django.utils.timezone import now + +from .models import DeveloperKey, UserProfile +from main.models import PackageFile +from packages.models import PackageRelation, Depend + +class DeveloperReport(object): + def __init__(self, slug, name, desc, packages_func, + names=None, attrs=None): + self.slug = slug + self.name = name + self.description = desc + self.packages = packages_func + self.names = names + self.attrs = attrs + + +def old(packages, username): + cutoff = now() - timedelta(days=365 * 2) + return packages.filter( + build_date__lt=cutoff).order_by('build_date') + + +def outofdate(packages, username): + cutoff = now() - timedelta(days=30) + return packages.filter( + flag_date__lt=cutoff).order_by('flag_date') + + +def big(packages, username): + cutoff = 50 * 1024 * 1024 + packages = packages.filter( + compressed_size__gte=cutoff).order_by('-compressed_size') + # Format the compressed and installed sizes with MB/GB/etc suffixes + for package in packages: + package.compressed_size_pretty = filesizeformat( + package.compressed_size) + package.installed_size_pretty = filesizeformat( + package.installed_size) + return packages + + +def badcompression(packages, username): + cutoff = 0.90 * F('installed_size') + packages = packages.filter(compressed_size__gt=0, installed_size__gt=0, + compressed_size__gte=cutoff).order_by('-compressed_size') + + # Format the compressed and installed sizes with MB/GB/etc suffixes + for package in packages: + package.compressed_size_pretty = filesizeformat( + package.compressed_size) + package.installed_size_pretty = filesizeformat( + package.installed_size) + ratio = package.compressed_size / float(package.installed_size) + package.ratio = '%.3f' % ratio + package.compress_type = package.filename.split('.')[-1] + + return packages + + +def uncompressed_man(packages, username): + # checking for all '.0'...'.9' + '.n' extensions + bad_files = PackageFile.objects.filter(is_directory=False, + directory__contains='/man/', + filename__regex=r'\.[0-9n]').exclude( + filename__endswith='.gz').exclude( + filename__endswith='.xz').exclude( + filename__endswith='.bz2').exclude( + filename__endswith='.html') + if username: + pkg_ids = set(packages.values_list('id', flat=True)) + bad_files = bad_files.filter(pkg__in=pkg_ids) + bad_files = bad_files.values_list( + 'pkg_id', flat=True).order_by().distinct() + return packages.filter(id__in=set(bad_files)) + + +def uncompressed_info(packages, username): + # we don't worry about looking for '*.info-1', etc., given that an + # uncompressed root page probably exists in the package anyway + bad_files = PackageFile.objects.filter(is_directory=False, + directory__endswith='/info/', filename__endswith='.info') + if username: + pkg_ids = set(packages.values_list('id', flat=True)) + bad_files = bad_files.filter(pkg__in=pkg_ids) + bad_files = bad_files.values_list( + 'pkg_id', flat=True).order_by().distinct() + return packages.filter(id__in=set(bad_files)) + + +def unneeded_orphans(packages, username): + owned = PackageRelation.objects.all().values('pkgbase') + required = Depend.objects.all().values('name') + # The two separate calls to exclude is required to do the right thing + return packages.exclude(pkgbase__in=owned).exclude( + pkgname__in=required) + + +def mismatched_signature(packages, username): + cutoff = timedelta(hours=24) + filtered = [] + packages = packages.select_related( + 'arch', 'repo', 'packager').filter(signature_bytes__isnull=False) + known_keys = DeveloperKey.objects.select_related( + 'owner').filter(owner__isnull=False) + known_keys = {dk.key: dk for dk in known_keys} + for package in packages: + bad = False + sig = package.signature + sig_date = sig.creation_time.replace(tzinfo=pytz.utc) + package.sig_date = sig_date.date() + dev_key = known_keys.get(sig.key_id, None) + if dev_key: + package.sig_by = dev_key.owner + if dev_key.owner_id != package.packager_id: + bad = True + else: + package.sig_by = sig.key_id + bad = True + + if sig_date > package.build_date + cutoff: + bad = True + + if bad: + filtered.append(package) + return filtered + + +REPORT_OLD = DeveloperReport('old', 'Old', + 'Packages last built more than two years ago', old) + +REPORT_OUTOFDATE = DeveloperReport('long-out-of-date', 'Long Out-of-date', + 'Packages marked out-of-date more than 30 days ago', outofdate) + +REPORT_BIG = DeveloperReport('big', 'Big', + 'Packages with compressed size > 50 MiB', big, + ['Compressed Size', 'Installed Size'], + ['compressed_size_pretty', 'installed_size_pretty']) + +REPORT_BADCOMPRESS = DeveloperReport('badcompression', 'Bad Compression', + 'Packages that have little need for compression', badcompression, + ['Compressed Size', 'Installed Size', 'Ratio', 'Type'], + ['compressed_size_pretty', 'installed_size_pretty','ratio', 'compress_type']) + + +REPORT_MAN = DeveloperReport('uncompressed-man', 'Uncompressed Manpages', + 'Packages with uncompressed manpages', uncompressed_man) + +REPORT_INFO = DeveloperReport('uncompressed-info', 'Uncompressed Info Pages', + 'Packages with uncompressed infopages', uncompressed_info) + +REPORT_ORPHANS = DeveloperReport('unneeded-orphans', 'Unneeded Orphans', + 'Orphan packages required by no other packages', unneeded_orphans) + +REPORT_SIGNATURE = DeveloperReport('mismatched-signature', 'Mismatched Signatures', + 'Packages with mismatched signatures', mismatched_signature, + ['Signature Date', 'Signed By', 'Packager'], + ['sig_date', 'sig_by', 'packager']) + + +def available_reports(): + return ( + REPORT_OLD, + REPORT_OUTOFDATE, + REPORT_BIG, + REPORT_BADCOMPRESS, + REPORT_MAN, + REPORT_INFO, + REPORT_ORPHANS, + REPORT_SIGNATURE, + ) diff --git a/devel/views.py b/devel/views.py index 1e20a437..cd2e25f8 100644 --- a/devel/views.py +++ b/devel/views.py @@ -1,6 +1,5 @@ from datetime import timedelta import operator -import pytz import time from django.http import HttpResponseRedirect @@ -10,21 +9,21 @@ from django.contrib.admin.models import LogEntry, ADDITION from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType from django.db import transaction -from django.db.models import F, Count, Max +from django.db.models import Count, Max from django.http import Http404 from django.shortcuts import get_object_or_404, render -from django.template.defaultfilters import filesizeformat from django.views.decorators.cache import never_cache from django.utils.encoding import force_unicode from django.utils.http import http_date from django.utils.timezone import now from .forms import ProfileForm, UserProfileForm, NewUserForm -from .models import DeveloperKey, UserProfile -from main.models import Package, PackageFile +from .models import UserProfile +from .reports import available_reports +from main.models import Package from main.models import Arch, Repo from news.models import News -from packages.models import PackageRelation, Signoff, FlagRequest, Depend +from packages.models import PackageRelation, Signoff, FlagRequest from packages.utils import get_signoff_groups from todolists.models import TodolistPackage from todolists.utils import get_annotated_todolists @@ -180,10 +179,13 @@ def change_profile(request): @login_required def report(request, report_name, username=None): - title = 'Developer Report' - packages = Package.objects.normal() - names = attrs = user = None + available = {report.slug: report for report in available_reports()} + report = available.get(report_name, None) + if report is None: + raise Http404 + packages = Package.objects.normal() + user = None if username: user = get_object_or_404(User, username=username, is_active=True) maintained = PackageRelation.objects.filter(user=user, @@ -193,126 +195,17 @@ def report(request, report_name, username=None): maints = User.objects.filter(id__in=PackageRelation.objects.filter( type=PackageRelation.MAINTAINER).values('user')) - if report_name == 'old': - title = 'Packages last built more than two years ago' - cutoff = now() - timedelta(days=365 * 2) - packages = packages.filter( - build_date__lt=cutoff).order_by('build_date') - elif report_name == 'long-out-of-date': - title = 'Packages marked out-of-date more than 30 days ago' - cutoff = now() - timedelta(days=30) - packages = packages.filter( - flag_date__lt=cutoff).order_by('flag_date') - elif report_name == 'big': - title = 'Packages with compressed size > 50 MiB' - cutoff = 50 * 1024 * 1024 - packages = packages.filter( - compressed_size__gte=cutoff).order_by('-compressed_size') - names = [ 'Compressed Size', 'Installed Size' ] - attrs = [ 'compressed_size_pretty', 'installed_size_pretty' ] - # Format the compressed and installed sizes with MB/GB/etc suffixes - for package in packages: - package.compressed_size_pretty = filesizeformat( - package.compressed_size) - package.installed_size_pretty = filesizeformat( - package.installed_size) - elif report_name == 'badcompression': - title = 'Packages that have little need for compression' - cutoff = 0.90 * F('installed_size') - packages = packages.filter(compressed_size__gt=0, installed_size__gt=0, - compressed_size__gte=cutoff).order_by('-compressed_size') - names = [ 'Compressed Size', 'Installed Size', 'Ratio', 'Type' ] - attrs = [ 'compressed_size_pretty', 'installed_size_pretty', - 'ratio', 'compress_type' ] - # Format the compressed and installed sizes with MB/GB/etc suffixes - for package in packages: - package.compressed_size_pretty = filesizeformat( - package.compressed_size) - package.installed_size_pretty = filesizeformat( - package.installed_size) - ratio = package.compressed_size / float(package.installed_size) - package.ratio = '%.3f' % ratio - package.compress_type = package.filename.split('.')[-1] - elif report_name == 'uncompressed-man': - title = 'Packages with uncompressed manpages' - # checking for all '.0'...'.9' + '.n' extensions - bad_files = PackageFile.objects.filter(is_directory=False, - directory__contains='/man/', - filename__regex=r'\.[0-9n]').exclude( - filename__endswith='.gz').exclude( - filename__endswith='.xz').exclude( - filename__endswith='.bz2').exclude( - filename__endswith='.html') - if username: - pkg_ids = set(packages.values_list('id', flat=True)) - bad_files = bad_files.filter(pkg__in=pkg_ids) - bad_files = bad_files.values_list( - 'pkg_id', flat=True).order_by().distinct() - packages = packages.filter(id__in=set(bad_files)) - elif report_name == 'uncompressed-info': - title = 'Packages with uncompressed infopages' - # we don't worry about looking for '*.info-1', etc., given that an - # uncompressed root page probably exists in the package anyway - bad_files = PackageFile.objects.filter(is_directory=False, - directory__endswith='/info/', filename__endswith='.info') - if username: - pkg_ids = set(packages.values_list('id', flat=True)) - bad_files = bad_files.filter(pkg__in=pkg_ids) - bad_files = bad_files.values_list( - 'pkg_id', flat=True).order_by().distinct() - packages = packages.filter(id__in=set(bad_files)) - elif report_name == 'unneeded-orphans': - title = 'Orphan packages required by no other packages' - owned = PackageRelation.objects.all().values('pkgbase') - required = Depend.objects.all().values('name') - # The two separate calls to exclude is required to do the right thing - packages = packages.exclude(pkgbase__in=owned).exclude( - pkgname__in=required) - elif report_name == 'mismatched-signature': - title = 'Packages with mismatched signatures' - names = [ 'Signature Date', 'Signed By', 'Packager' ] - attrs = [ 'sig_date', 'sig_by', 'packager' ] - cutoff = timedelta(hours=24) - filtered = [] - packages = packages.select_related( - 'arch', 'repo', 'packager').filter(signature_bytes__isnull=False) - known_keys = DeveloperKey.objects.select_related( - 'owner').filter(owner__isnull=False) - known_keys = {dk.key: dk for dk in known_keys} - for package in packages: - bad = False - sig = package.signature - sig_date = sig.creation_time.replace(tzinfo=pytz.utc) - package.sig_date = sig_date.date() - dev_key = known_keys.get(sig.key_id, None) - if dev_key: - package.sig_by = dev_key.owner - if dev_key.owner_id != package.packager_id: - bad = True - else: - package.sig_by = sig.key_id - bad = True - - if sig_date > package.build_date + cutoff: - bad = True - - if bad: - filtered.append(package) - packages = filtered - else: - raise Http404 - arches = {pkg.arch for pkg in packages} repos = {pkg.repo for pkg in packages} context = { 'all_maintainers': maints, - 'title': title, + 'title': report.description, 'maintainer': user, - 'packages': packages, + 'packages': report.packages(packages, username), 'arches': sorted(arches), 'repos': sorted(repos), - 'column_names': names, - 'column_attrs': attrs, + 'column_names': report.names, + 'column_attrs': report.attrs, } return render(request, 'devel/packages.html', context) -- cgit v1.2.3-2-g168b From 871b284dd76f01d8a297f43ae86d869e2b7d2b3c Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 22 Feb 2014 13:48:52 -0600 Subject: Generate list of reports dynamically Signed-off-by: Dan McGee --- devel/reports.py | 15 ++++++++++----- devel/views.py | 3 ++- 2 files changed, 12 insertions(+), 6 deletions(-) (limited to 'devel') diff --git a/devel/reports.py b/devel/reports.py index f1ffab3f..53cf9e69 100644 --- a/devel/reports.py +++ b/devel/reports.py @@ -11,13 +11,14 @@ from packages.models import PackageRelation, Depend class DeveloperReport(object): def __init__(self, slug, name, desc, packages_func, - names=None, attrs=None): + names=None, attrs=None, personal=True): self.slug = slug self.name = name self.description = desc self.packages = packages_func self.names = names self.attrs = attrs + self.personal = personal def old(packages, username): @@ -143,7 +144,7 @@ REPORT_BIG = DeveloperReport('big', 'Big', ['compressed_size_pretty', 'installed_size_pretty']) REPORT_BADCOMPRESS = DeveloperReport('badcompression', 'Bad Compression', - 'Packages that have little need for compression', badcompression, + 'Packages with a compression ratio of less than 10%', badcompression, ['Compressed Size', 'Installed Size', 'Ratio', 'Type'], ['compressed_size_pretty', 'installed_size_pretty','ratio', 'compress_type']) @@ -152,13 +153,17 @@ REPORT_MAN = DeveloperReport('uncompressed-man', 'Uncompressed Manpages', 'Packages with uncompressed manpages', uncompressed_man) REPORT_INFO = DeveloperReport('uncompressed-info', 'Uncompressed Info Pages', - 'Packages with uncompressed infopages', uncompressed_info) + 'Packages with uncompressed info pages', uncompressed_info) REPORT_ORPHANS = DeveloperReport('unneeded-orphans', 'Unneeded Orphans', - 'Orphan packages required by no other packages', unneeded_orphans) + 'Packages that have no maintainer and are not required by any ' + + 'other package in any repository', unneeded_orphans, + personal=False) REPORT_SIGNATURE = DeveloperReport('mismatched-signature', 'Mismatched Signatures', - 'Packages with mismatched signatures', mismatched_signature, + 'Packages where 1) signing key is unknown, 2) signer != packager, ' + + 'or 3) signature timestamp more than 24 hours after build timestamp', + mismatched_signature, ['Signature Date', 'Signed By', 'Packager'], ['sig_date', 'sig_by', 'packager']) diff --git a/devel/views.py b/devel/views.py index cd2e25f8..c53da668 100644 --- a/devel/views.py +++ b/devel/views.py @@ -57,7 +57,8 @@ def index(request): 'todos': todolists, 'flagged': flagged, 'todopkgs': todopkgs, - 'signoffs': signoffs + 'signoffs': signoffs, + 'reports': available_reports(), } return render(request, 'devel/index.html', page_dict) -- cgit v1.2.3-2-g168b From e395bef7241a71cd72d24362bf233251746c8c8b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 22 Feb 2014 13:56:51 -0600 Subject: Split signature report into two reports Signed-off-by: Dan McGee --- devel/reports.py | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) (limited to 'devel') diff --git a/devel/reports.py b/devel/reports.py index 53cf9e69..ca1a3321 100644 --- a/devel/reports.py +++ b/devel/reports.py @@ -103,7 +103,6 @@ def unneeded_orphans(packages, username): def mismatched_signature(packages, username): - cutoff = timedelta(hours=24) filtered = [] packages = packages.select_related( 'arch', 'repo', 'packager').filter(signature_bytes__isnull=False) @@ -113,8 +112,6 @@ def mismatched_signature(packages, username): for package in packages: bad = False sig = package.signature - sig_date = sig.creation_time.replace(tzinfo=pytz.utc) - package.sig_date = sig_date.date() dev_key = known_keys.get(sig.key_id, None) if dev_key: package.sig_by = dev_key.owner @@ -124,14 +121,26 @@ def mismatched_signature(packages, username): package.sig_by = sig.key_id bad = True - if sig_date > package.build_date + cutoff: - bad = True - if bad: filtered.append(package) return filtered +def signature_time(packages, username): + cutoff = timedelta(hours=24) + filtered = [] + packages = packages.select_related( + 'arch', 'repo', 'packager').filter(signature_bytes__isnull=False) + for package in packages: + sig = package.signature + sig_date = sig.creation_time.replace(tzinfo=pytz.utc) + package.sig_date = sig_date.date() + if sig_date > package.build_date + cutoff: + filtered.append(package) + + return filtered + + REPORT_OLD = DeveloperReport('old', 'Old', 'Packages last built more than two years ago', old) @@ -160,12 +169,19 @@ REPORT_ORPHANS = DeveloperReport('unneeded-orphans', 'Unneeded Orphans', + 'other package in any repository', unneeded_orphans, personal=False) -REPORT_SIGNATURE = DeveloperReport('mismatched-signature', 'Mismatched Signatures', - 'Packages where 1) signing key is unknown, 2) signer != packager, ' - + 'or 3) signature timestamp more than 24 hours after build timestamp', +REPORT_SIGNATURE = DeveloperReport('mismatched-signature', + 'Mismatched Signatures', + 'Packages where the signing key is unknown or signer != packager', mismatched_signature, - ['Signature Date', 'Signed By', 'Packager'], - ['sig_date', 'sig_by', 'packager']) + ['Signed By', 'Packager'], + ['sig_by', 'packager']) + +REPORT_SIG_TIME = DeveloperReport('signature-time', 'Signature Time', + 'Packages where the signature timestamp is more than 24 hours ' + + 'after the build timestamp', + signature_time, + ['Signature Date', 'Packager'], + ['sig_date', 'packager']) def available_reports(): @@ -178,4 +194,5 @@ def available_reports(): REPORT_INFO, REPORT_ORPHANS, REPORT_SIGNATURE, + REPORT_SIG_TIME, ) -- cgit v1.2.3-2-g168b From 268317dd331bcdbe1c2828583034c0274a19eea3 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 9 Mar 2014 11:58:51 -0500 Subject: Use localStorage to save/restore developer report filters Signed-off-by: Dan McGee --- devel/views.py | 1 + 1 file changed, 1 insertion(+) (limited to 'devel') diff --git a/devel/views.py b/devel/views.py index c53da668..972d0abb 100644 --- a/devel/views.py +++ b/devel/views.py @@ -201,6 +201,7 @@ def report(request, report_name, username=None): context = { 'all_maintainers': maints, 'title': report.description, + 'report': report, 'maintainer': user, 'packages': report.packages(packages, username), 'arches': sorted(arches), -- cgit v1.2.3-2-g168b