diff options
| -rw-r--r-- | devel/utils.py | 17 | ||||
| -rw-r--r-- | devel/views.py | 23 | ||||
| -rw-r--r-- | main/models.py | 2 | ||||
| -rw-r--r-- | mirrors/utils.py | 34 | ||||
| -rw-r--r-- | packages/urls.py | 1 | ||||
| -rw-r--r-- | packages/utils.py | 37 | ||||
| -rw-r--r-- | packages/views/__init__.py | 54 | ||||
| -rw-r--r-- | packages/views/search.py | 28 | ||||
| -rw-r--r-- | sitemaps.py | 4 | ||||
| -rw-r--r-- | sitestatic/archweb.css | 17 | ||||
| -rw-r--r-- | templates/devel/index.html | 9 | ||||
| -rw-r--r-- | templates/packages/details.html | 2 | ||||
| -rw-r--r-- | templates/packages/flag.html | 2 | ||||
| -rw-r--r-- | templates/packages/flaghelp.html | 7 | ||||
| -rw-r--r-- | templates/public/index.html | 16 | ||||
| -rw-r--r-- | templates/public/keys.html | 23 | ||||
| -rw-r--r-- | todolists/utils.py | 6 |
17 files changed, 204 insertions, 78 deletions
diff --git a/devel/utils.py b/devel/utils.py index ec035d13..85b4e42f 100644 --- a/devel/utils.py +++ b/devel/utils.py @@ -49,6 +49,7 @@ class UserFinder(object): self.cache = {} self.username_cache = {} self.email_cache = {} + self.pgp_cache = {} @staticmethod def user_email(name, email): @@ -146,9 +147,25 @@ class UserFinder(object): self.email_cache[email] = user return user + def find_by_pgp_key(self, pgp_key): + if not pgp_key: + return None + if pgp_key in self.pgp_cache: + return self.pgp_cache[pgp_key] + + try: + user = User.objects.get( + userprofile__pgp_key__endswith=pgp_key) + except User.DoesNotExist: + user = None + + self.pgp_cache[pgp_key] = user + return user + def clear_cache(self): self.cache = {} self.username_cache = {} self.email_cache = {} + self.pgp_cache = {} # vim: set ts=4 sw=4 et: diff --git a/devel/views.py b/devel/views.py index f2c3949c..2b003bf7 100644 --- a/devel/views.py +++ b/devel/views.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta +from datetime import date, datetime, timedelta import operator import pytz import random @@ -28,7 +28,7 @@ from main.utils import utc_now from packages.models import PackageRelation from packages.utils import get_signoff_groups from todolists.utils import get_annotated_todolists -from .utils import get_annotated_maintainers +from .utils import get_annotated_maintainers, UserFinder @login_required @@ -232,6 +232,25 @@ def report(request, report_name, username=None): # The two separate calls to exclude is required to do the right thing packages = packages.exclude(pkgbase__in=owned).exclude( pkgname__in=required) + elif report_name == 'mismatched-signature': + title = 'Packages with mismatched signatures' + names = [ 'Signature Date', 'Signed By', 'Packager' ] + attrs = [ 'sig_date', 'sig_by', 'packager' ] + cutoff = timedelta(hours=24) + finder = UserFinder() + filtered = [] + packages = packages.filter(pgp_signature__isnull=False) + for package in packages: + sig_date = package.signature.datetime.replace(tzinfo=pytz.utc) + package.sig_date = sig_date.date() + key_id = package.signature.key_id + signer = finder.find_by_pgp_key(key_id) + package.sig_by = signer or key_id + if signer is None or signer.id != package.packager_id: + filtered.append(package) + elif sig_date > package.build_date + cutoff: + filtered.append(package) + packages = filtered else: raise Http404 diff --git a/main/models.py b/main/models.py index 34cbcd17..db926dda 100644 --- a/main/models.py +++ b/main/models.py @@ -115,7 +115,7 @@ class Repo(models.Model): help_text="Is this repo meant for package staging?") bugs_project = models.SmallIntegerField(default=1, help_text="Flyspray project ID for this repository.") - bugs_category = models.SmallIntegerField(default=0, + bugs_category = models.SmallIntegerField(default=2, help_text="Flyspray category ID for this repository.") svn_root = models.CharField(max_length=64, help_text="SVN root (e.g. path) for this repository.") diff --git a/mirrors/utils.py b/mirrors/utils.py index 0f8fef84..619d5f5c 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -27,6 +27,7 @@ def annotate_url(url, delays): url.delay = None url.score = None + @cache_function(123) def get_mirror_statuses(cutoff=default_cutoff): cutoff_time = utc_now() - cutoff @@ -81,6 +82,7 @@ def get_mirror_statuses(cutoff=default_cutoff): 'urls': urls, } + @cache_function(117) def get_mirror_errors(cutoff=default_cutoff): cutoff_time = utc_now() - cutoff @@ -96,4 +98,36 @@ def get_mirror_errors(cutoff=default_cutoff): err['country'] = err['url__country'] or err['url__mirror__country'] return errors + +@cache_function(295) +def get_mirror_url_for_download(cutoff=default_cutoff): + '''Find a good mirror URL to use for package downloads. If we have mirror + status data available, it is used to determine a good choice by looking at + the last batch of status rows.''' + cutoff_time = utc_now() - cutoff + status_data = MirrorLog.objects.filter( + check_time__gte=cutoff_time).aggregate( + Max('check_time'), Max('last_sync')) + if status_data: + min_check_time = status_data['check_time__max'] - timedelta(minutes=5) + min_sync_time = status_data['last_sync__max'] - timedelta(minutes=30) + best_logs = MirrorLog.objects.filter(is_success=True, + check_time__gte=min_check_time, last_sync__gte=min_sync_time, + url__mirror__public=True, url__mirror__active=True, + url__protocol__protocol__iexact='HTTP').order_by( + 'duration')[:1] + if best_logs: + return MirrorUrl.objects.get(id=best_logs[0].url_id) + + mirror_urls = MirrorUrl.objects.filter( + mirror__public=True, mirror__active=True, + protocol__protocol__iexact='HTTP') + # look first for an 'Any' URL, then fall back to any HTTP URL + filtered_urls = mirror_urls.filter(mirror__country='Any')[:1] + if not filtered_urls: + filtered_urls = mirror_urls[:1] + if not filtered_urls: + return None + return filtered_urls[0] + # vim: set ts=4 sw=4 et: diff --git a/packages/urls.py b/packages/urls.py index 9a063f43..1fd54a7e 100644 --- a/packages/urls.py +++ b/packages/urls.py @@ -22,6 +22,7 @@ urlpatterns = patterns('packages.views', (r'^update/$', 'update'), (r'^$', 'search', {}, 'packages-search'), + (r'^search/json/$', 'search_json'), (r'^(?P<page>\d+)/$', 'search'), (r'^differences/$', 'arch_differences', {}, 'packages-differences'), diff --git a/packages/utils.py b/packages/utils.py index ab3c074f..55a2901a 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -2,13 +2,15 @@ from collections import defaultdict from itertools import chain from operator import itemgetter +from django.core.serializers.json import DjangoJSONEncoder from django.db import connection from django.db.models import Count, Max, F from django.contrib.auth.models import User -from main.models import Package, Arch, Repo +from main.models import Package, PackageDepend, PackageFile, Arch, Repo from main.utils import cache_function, groupby_preserve_order, PackageStandin from .models import (PackageGroup, PackageRelation, + License, Conflict, Provision, Replacement, SignoffSpecification, Signoff, DEFAULT_SIGNOFF_SPEC) @cache_function(127) @@ -426,4 +428,37 @@ def get_signoff_groups(repos=None, user=None): return signoff_groups + +class PackageJSONEncoder(DjangoJSONEncoder): + pkg_attributes = [ 'pkgname', 'pkgbase', 'repo', 'arch', 'pkgver', + 'pkgrel', 'epoch', 'pkgdesc', 'url', 'filename', 'compressed_size', + 'installed_size', 'build_date', 'last_update', 'flag_date', + 'maintainers', 'packager' ] + pkg_list_attributes = [ 'groups', 'licenses', 'conflicts', + 'provides', 'replaces', 'depends' ] + + def default(self, obj): + if hasattr(obj, '__iter__'): + # mainly for queryset serialization + return list(obj) + if isinstance(obj, Package): + data = dict((attr, getattr(obj, attr)) + for attr in self.pkg_attributes) + for attr in self.pkg_list_attributes: + data[attr] = getattr(obj, attr).all() + return data + if isinstance(obj, PackageFile): + filename = obj.filename or '' + return obj.directory + filename + if isinstance(obj, (Repo, Arch)): + return obj.name.lower() + if isinstance(obj, (PackageGroup, License)): + return obj.name + if isinstance(obj, (Conflict, Provision, Replacement, PackageDepend)): + return unicode(obj) + elif isinstance(obj, User): + return obj.username + return super(PackageJSONEncoder, self).default(obj) + + # vim: set ts=4 sw=4 et: diff --git a/packages/views/__init__.py b/packages/views/__init__.py index aa2721af..fc132105 100644 --- a/packages/views/__init__.py +++ b/packages/views/__init__.py @@ -4,7 +4,6 @@ from urllib import urlencode from django.contrib import messages from django.contrib.auth.decorators import permission_required from django.contrib.auth.models import User -from django.core.serializers.json import DjangoJSONEncoder from django.http import HttpResponse, Http404 from django.shortcuts import get_object_or_404, redirect from django.utils import simplejson @@ -12,50 +11,19 @@ from django.views.decorators.http import require_POST from django.views.decorators.vary import vary_on_headers from django.views.generic.simple import direct_to_template -from main.models import Package, PackageFile, PackageDepend, Arch, Repo +from main.models import Package, PackageFile, Arch, Repo from mirrors.models import MirrorUrl -from ..models import (PackageRelation, PackageGroup, License, - Conflict, Provision, Replacement) +from mirrors.utils import get_mirror_url_for_download +from ..models import PackageRelation from ..utils import (get_group_info, get_differences_info, - multilib_differences, get_wrong_permissions) + multilib_differences, get_wrong_permissions, PackageJSONEncoder) # make other views available from this same package from .flag import flaghelp, flag, flag_confirmed, unflag, unflag_all -from .search import search +from .search import search, search_json from .signoff import signoffs, signoff_package, signoff_options, signoffs_json -class PackageJSONEncoder(DjangoJSONEncoder): - pkg_attributes = [ 'pkgname', 'pkgbase', 'repo', 'arch', 'pkgver', - 'pkgrel', 'epoch', 'pkgdesc', 'url', 'filename', 'compressed_size', - 'installed_size', 'build_date', 'last_update', 'flag_date', - 'maintainers', 'packager' ] - pkg_list_attributes = [ 'groups', 'licenses', 'conflicts', - 'provides', 'replaces', 'depends' ] - - def default(self, obj): - if hasattr(obj, '__iter__'): - # mainly for queryset serialization - return list(obj) - if isinstance(obj, Package): - data = dict((attr, getattr(obj, attr)) - for attr in self.pkg_attributes) - for attr in self.pkg_list_attributes: - data[attr] = getattr(obj, attr).all() - return data - if isinstance(obj, PackageFile): - filename = obj.filename or '' - return obj.directory + filename - if isinstance(obj, (Repo, Arch)): - return obj.name.lower() - if isinstance(obj, (PackageGroup, License)): - return obj.name - if isinstance(obj, (Conflict, Provision, Replacement, PackageDepend)): - return unicode(obj) - elif isinstance(obj, User): - return obj.username - return super(PackageJSONEncoder, self).default(obj) - def opensearch(request): if request.is_secure(): domain = "https://%s" % request.META['HTTP_HOST'] @@ -225,21 +193,15 @@ def files_json(request, name, repo, arch): def download(request, name, repo, arch): pkg = get_object_or_404(Package, pkgname=name, repo__name__iexact=repo, arch__name=arch) - mirror_urls = MirrorUrl.objects.filter( - mirror__public=True, mirror__active=True, - protocol__protocol__iexact='HTTP') - # look first for an 'Any' URL, then fall back to any HTTP URL - filtered_urls = mirror_urls.filter(mirror__country='Any')[:1] - if not filtered_urls: - filtered_urls = mirror_urls[:1] - if not filtered_urls: + url = get_mirror_url_for_download() + if not url: raise Http404 arch = pkg.arch.name if pkg.arch.agnostic: # grab the first non-any arch to fake the download path arch = Arch.objects.exclude(agnostic=True)[0].name values = { - 'host': filtered_urls[0].url, + 'host': url.url, 'arch': arch, 'repo': pkg.repo.name.lower(), 'file': pkg.filename, diff --git a/packages/views/search.py b/packages/views/search.py index 1431893d..a09de0a7 100644 --- a/packages/views/search.py +++ b/packages/views/search.py @@ -4,11 +4,14 @@ from django import forms from django.contrib.admin.widgets import AdminDateWidget from django.contrib.auth.models import User from django.db.models import Q +from django.http import HttpResponse from django.views.generic import list_detail +from django.utils import simplejson from main.models import Package, Arch, Repo from main.utils import make_choice from ..models import PackageRelation +from ..utils import PackageJSONEncoder def coerce_limit_value(value): @@ -101,7 +104,7 @@ def parse_form(form, packages): if form.cleaned_data['name']: name = form.cleaned_data['name'] - packages = packages.filter(pkgname__icontains=name) + packages = packages.filter(pkgname=name) if form.cleaned_data['desc']: desc = form.cleaned_data['desc'] @@ -157,4 +160,27 @@ def search(request, page=None): template_object_name="package", extra_context=page_dict) + +def search_json(request): + limit = 250 + + container = { + 'version': 2, + 'limit': limit, + 'valid': False, + 'results': [], + } + + if request.GET: + form = PackageSearchForm(data=request.GET) + if form.is_valid(): + packages = Package.objects.normal() + packages = parse_form(form, packages)[:limit] + container['results'] = packages + container['valid'] = True + + to_json = simplejson.dumps(container, ensure_ascii=False, + cls=PackageJSONEncoder) + return HttpResponse(to_json, mimetype='application/json') + # vim: set ts=4 sw=4 et: diff --git a/sitemaps.py b/sitemaps.py index 8e9ba36f..424168bf 100644 --- a/sitemaps.py +++ b/sitemaps.py @@ -13,7 +13,7 @@ class PackagesSitemap(Sitemap): priority = "0.5" def items(self): - return Package.objects.normal() + return Package.objects.all().order_by() def lastmod(self, obj): return obj.last_update @@ -68,7 +68,7 @@ class NewsSitemap(Sitemap): self.one_week_ago = now - timedelta(days=7) def items(self): - return News.objects.all() + return News.objects.all().order_by() def lastmod(self, obj): return obj.last_modified diff --git a/sitestatic/archweb.css b/sitestatic/archweb.css index ced1fd2a..9e531674 100644 --- a/sitestatic/archweb.css +++ b/sitestatic/archweb.css @@ -258,6 +258,23 @@ table.pretty2 { border: 1px solid #bbb; } + /* additional styles for JS sorting */ + table.pretty2 th.header { + padding-right: 20px; + background-image: url(nosort.gif); + background-repeat: no-repeat; + background-position: center right; + cursor: pointer; + } + + table.pretty2 th.headerSortDown { + background-image: url(desc.gif); + } + + table.pretty2 th.headerSortUp { + background-image: url(asc.gif); + } + table.pretty2 td { padding: 0.35em; border: 1px dotted #bbb; diff --git a/templates/devel/index.html b/templates/devel/index.html index 3318da58..cda510e4 100644 --- a/templates/devel/index.html +++ b/templates/devel/index.html @@ -147,9 +147,6 @@ <h3>Developer Reports</h3> <ul> - <li><a href="reports/big/">Big</a>: - All packages with compressed size > 50 MiB - (<a href="reports/big/{{ user.username }}/">yours only</a>)</li> <li><a href="reports/old/">Old</a>: Packages last built more than two years ago (<a href="reports/old/{{ user.username }}/">yours only</a>)</li> @@ -162,6 +159,12 @@ <li><a href="reports/uncompressed-info/">Uncompressed Info Pages</a>: Self-explanatory (<a href="reports/uncompressed-info/{{ user.username }}/">yours only</a>)</li> + <li><a href="reports/mismatched-signature/">Mismatched Signatures</a>: + Packages where 1) signing key is unknown, 2) signer != packager, or 3) signature timestamp more than 24 hours after build timestamp + (<a href="reports/mismatched-signature/{{ user.username }}/">yours only</a>)</li> + <li><a href="reports/big/">Big</a>: + All packages with compressed size > 50 MiB + (<a href="reports/big/{{ user.username }}/">yours only</a>)</li> <li><a href="reports/badcompression/">Bad Compression</a>: Packages with a compression ratio of less than 10% (<a href="reports/badcompression/{{ user.username }}/">yours only</a>)</li> diff --git a/templates/packages/details.html b/templates/packages/details.html index 7a83f9ee..00b2c70c 100644 --- a/templates/packages/details.html +++ b/templates/packages/details.html @@ -146,7 +146,7 @@ <td>{{ pkg.build_date|date:"DATETIME_FORMAT" }} UTC</td> </tr>{% if pkg.signature %}<tr> <th>Signed By:</th> - <td>{% with pkg.signer as signer %}{% if signer %}{% pgp_key_link pkg.signature.key_id signer.get_full_name %}{% else %}Unknown{% endif %}{% endwith %}</td> + <td>{% with pkg.signer as signer %}{% if signer %}{% pgp_key_link pkg.signature.key_id signer.get_full_name %}{% else %}Unknown ({% pgp_key_link pkg.signature.key_id %}){% endif %}{% endwith %}</td> </tr><tr> <th>Signature Date:</th> <td>{{ pkg.signature.datetime|date:"DATETIME_FORMAT" }} UTC</td> diff --git a/templates/packages/flag.html b/templates/packages/flag.html index 79920c12..f64622c9 100644 --- a/templates/packages/flag.html +++ b/templates/packages/flag.html @@ -24,7 +24,7 @@ for short messages only. If you need more than 200 characters for your message, then file a bug report, email the maintainer directly, or send an email to the <a href="https://lists.parabolagnulinux.org/mailman/listinfo/dev" - title="Visit the dev mailing list">Parabola Development mailing list</a> + title="Visit the dev mailing list">Parabola dev mailing list</a> with your additional text.</p> <p><strong>Note:</strong> Do <em>not</em> use this facility if the diff --git a/templates/packages/flaghelp.html b/templates/packages/flaghelp.html index d60018fa..3320bf2a 100644 --- a/templates/packages/flaghelp.html +++ b/templates/packages/flaghelp.html @@ -1,6 +1,5 @@ -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" - "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> -<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> +<!DOCTYPE html> +<html lang="en"> <head> <title>Flagging Packages</title> <meta http-equiv="content-type" content="text/html; charset=utf-8" /> @@ -26,7 +25,7 @@ for short messages only. If you need more than 200 characters for your message, then file a bug report, email the maintainer directly, or send an email to the <a target="_blank" href="https://lists.parabolagnulinux.org/mailman/listinfo/dev" - title="Visit the parabola dev mailing list">parabola mailing list</a> + title="Visit the parabola dev mailing list">Parabola dev mailing list</a> with your additional text.</p> <p><strong>Note:</strong> Please do <em>not</em> use this facility if the diff --git a/templates/public/index.html b/templates/public/index.html index 58d5815d..ebfcde27 100644 --- a/templates/public/index.html +++ b/templates/public/index.html @@ -21,11 +21,15 @@ <p>Currently we provide the official packages from Arch that meet the <a href="http://www.gnu.org/distros/free-system-distribution-guidelines.html">FSDG</a>, replacements for those that don't, and our own additions, all of - them optimized for the i686, x86-64, and mips64el - architectures.</p> + them optimized for the i686, x86-64, and mips64el architectures.</p> - <p>You can find us on IRC at <a - href="irc://irc.freenode.net/#parabola">irc.freenode.net/#parabola</a>.</p> + <p>Our strong community is diverse and helpful. Please + check out our <a href="irc://chat.freenode.net#parabola" title="IRC channel">IRC channel</a> + and <a href="https://lists.parabolagnulinux.org/mailman/listinfo/" + title="Parabola Mailing Lists">mailing lists</a> + to get your feet wet. Also glance through our <a href="https://wiki.parabolagnulinux.org/" + title="Parabola Wiki">wiki</a> + if you want to learn more about Parabola.</p> <p class="readmore"><a href="{% url 'page-about' %}" title="Learn more about Parabola">Learn more...</a></p> @@ -121,13 +125,13 @@ <h4>Community</h4> <ul> - <li><a href="https://lists.parabolagnulinux.org/" + <li><a href="https://lists.parabolagnulinux.org/mailman/listinfo/" title="Community and developer mailing lists">Mailing Lists</a></li> <li><a href="https://lists.parabolagnulinux.org/pipermail/dev/" title="dev mailing list archives">Dev Archives</a></li> <li><a href="https://wiki.parabolagnulinux.org/IRC_Channels" title="Official and regional IRC communities">IRC Channels</a></li> - <li><a href="http://identi.ca/group/parabola" title="Parabola at identi.ca">Identi.ca group</a></li> + <li><a href="https://identi.ca/group/parabola" title="Parabola at identi.ca">Identi.ca group</a></li> </ul> <h4>Support</h4> diff --git a/templates/public/keys.html b/templates/public/keys.html index f0aa310e..257a16ae 100644 --- a/templates/public/keys.html +++ b/templates/public/keys.html @@ -1,4 +1,5 @@ {% extends "base.html" %} +{% load static from staticfiles %} {% load pgp %} {% block title %}Parabola - Master Signing Keys{% endblock %} @@ -66,17 +67,11 @@ <table class="pretty2" id="key-status"> <thead> <tr> - <th></th> + <th>Developer</th> <th>PGP Key</th> {% for key in keys %} - <th>{{ key.owner.get_full_name }}</th> - {% endfor %} - </tr> - <tr> - <th></th> - <th></th> - {% for key in keys %} - <th>{% pgp_key_link key.pgp_key %}</th> + <th>{{ key.owner.get_full_name }}<br/> + {% pgp_key_link key.pgp_key %}</th> {% endfor %} </tr> </thead> @@ -94,4 +89,14 @@ </tbody> </table> </div> +{% load cdn %}{% jquery %} +<script type="text/javascript" src="{% static "jquery.tablesorter.min.js" %}"></script> +<script type="text/javascript" src="{% static "archweb.js" %}"></script> +<script type="text/javascript"> +$(document).ready(function() { + $("#key-status").tablesorter({ + headers: { 1: { sorter: false } } + }); +}); +</script> {% endblock %} diff --git a/todolists/utils.py b/todolists/utils.py index 894f3f1d..24101e86 100644 --- a/todolists/utils.py +++ b/todolists/utils.py @@ -2,9 +2,13 @@ from django.db.models import Count from main.models import Todolist + def get_annotated_todolists(): qs = Todolist.objects.all() - lists = qs.select_related('creator').annotate( + lists = qs.select_related('creator').defer( + 'creator__email', 'creator__password', 'creator__is_staff', + 'creator__is_active', 'creator__is_superuser', + 'creator__last_login', 'creator__date_joined').annotate( pkg_count=Count('todolistpkg')).order_by('-date_added') incomplete = qs.filter(todolistpkg__complete=False).annotate( Count('todolistpkg')).values_list('id', 'todolistpkg__count') |