From 2c958511c41f53fb7de49ed4662eec966e0b76a5 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 22 Jan 2013 16:48:49 -0600 Subject: Use a subquery rather than two queries in attach_maintainers Now that we are using a database that doesn't stink, it makes more sense to do all of the stuff we need to do down at the database level. This helps a lot when 500+ packages are in play at a given time, such as some of our larger rebuild todo lists. Signed-off-by: Dan McGee --- packages/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'packages') diff --git a/packages/utils.py b/packages/utils.py index a72404f4..49aeb8ce 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -6,6 +6,7 @@ import re from django.core.serializers.json import DjangoJSONEncoder from django.db import connection from django.db.models import Count, Max, F +from django.db.models.query import QuerySet from django.contrib.auth.models import User from main.models import Package, PackageFile, Arch, Repo @@ -253,8 +254,11 @@ def attach_maintainers(packages): '''Given a queryset or something resembling it of package objects, find all the maintainers and attach them to the packages to prevent N+1 query cascading.''' - packages = list(packages) - pkgbases = {p.pkgbase for p in packages if p is not None} + if isinstance(packages, QuerySet): + pkgbases = packages.values('pkgbase') + else: + packages = list(packages) + pkgbases = {p.pkgbase for p in packages if p is not None} rels = PackageRelation.objects.filter(type=PackageRelation.MAINTAINER, pkgbase__in=pkgbases).values_list( 'pkgbase', 'user_id').order_by().distinct() -- cgit v1.2.3-2-g168b From 85dc9d6df9f9038be6049aefbf8eb9ea94f9f23c Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 22 Jan 2013 17:20:41 -0700 Subject: Override the default admin queryset for some models Because some attributes are optional or otherwise not auto-magically picked up by Django, we can help the performance of loading these pages a lot by forcing a select_related() on the queryset used by the admin. For something like signoff_specifications, this drops the query count from ~107 to 9 queries. Signed-off-by: Dan McGee --- packages/admin.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'packages') diff --git a/packages/admin.py b/packages/admin.py index 5e32dbb4..820bbb29 100644 --- a/packages/admin.py +++ b/packages/admin.py @@ -3,6 +3,7 @@ from django.contrib import admin from .models import (PackageRelation, FlagRequest, Signoff, SignoffSpecification, Update) + class PackageRelationAdmin(admin.ModelAdmin): list_display = ('pkgbase', 'user', 'type', 'created') list_filter = ('type', 'user') @@ -19,6 +20,10 @@ class FlagRequestAdmin(admin.ModelAdmin): ordering = ('-created',) date_hierarchy = 'created' + def queryset(self, request): + qs = super(FlagRequestAdmin, self).queryset(request) + return qs.select_related('repo', 'user') + class SignoffAdmin(admin.ModelAdmin): list_display = ('pkgbase', 'full_version', 'arch', 'repo', @@ -28,6 +33,7 @@ class SignoffAdmin(admin.ModelAdmin): ordering = ('-created',) date_hierarchy = 'created' + class SignoffSpecificationAdmin(admin.ModelAdmin): list_display = ('pkgbase', 'full_version', 'arch', 'repo', 'user', 'created', 'comments') @@ -36,6 +42,10 @@ class SignoffSpecificationAdmin(admin.ModelAdmin): ordering = ('-created',) date_hierarchy = 'created' + def queryset(self, request): + qs = super(SignoffSpecificationAdmin, self).queryset(request) + return qs.select_related('arch', 'repo', 'user') + class UpdateAdmin(admin.ModelAdmin): list_display = ('pkgname', 'repo', 'arch', 'action_flag', -- cgit v1.2.3-2-g168b From be49f26a815cca589c625ff8dd99c85a80262281 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 22 Jan 2013 20:58:50 -0700 Subject: Slight optimization when searching for removed package Signed-off-by: Dan McGee --- packages/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'packages') diff --git a/packages/models.py b/packages/models.py index ef86d8e9..ff677883 100644 --- a/packages/models.py +++ b/packages/models.py @@ -318,7 +318,8 @@ class Update(models.Model): return u'%s-%s' % (self.new_pkgver, self.new_pkgrel) def elsewhere(self): - return Package.objects.filter(pkgname=self.pkgname, arch=self.arch) + return Package.objects.normal().filter( + pkgname=self.pkgname, arch=self.arch) def __unicode__(self): return u'%s of %s on %s' % (self.get_action_flag_display(), -- cgit v1.2.3-2-g168b From a10798b756bbfc5d8dbad76546ca670efca75e56 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 23 Jan 2013 09:18:59 -0700 Subject: Use querysets for calls to get_object_or_404(Package) This works better in most cases since we need the architecture and repository objects at some point during the view process. Signed-off-by: Dan McGee --- packages/views/display.py | 8 ++++---- packages/views/flag.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'packages') diff --git a/packages/views/display.py b/packages/views/display.py index 445c1abe..c2369aba 100644 --- a/packages/views/display.py +++ b/packages/views/display.py @@ -168,7 +168,7 @@ def group_details(request, arch, name): def files(request, name, repo, arch): - pkg = get_object_or_404(Package, + pkg = get_object_or_404(Package.objects.normal(), pkgname=name, repo__name__iexact=repo, arch__name=arch) # files are inserted in sorted order, so preserve that fileslist = PackageFile.objects.filter(pkg=pkg).order_by('id') @@ -185,14 +185,14 @@ def files(request, name, repo, arch): def details_json(request, name, repo, arch): - pkg = get_object_or_404(Package, + pkg = get_object_or_404(Package.objects.normal(), pkgname=name, repo__name__iexact=repo, arch__name=arch) to_json = json.dumps(pkg, ensure_ascii=False, cls=PackageJSONEncoder) return HttpResponse(to_json, content_type='application/json') def files_json(request, name, repo, arch): - pkg = get_object_or_404(Package, + pkg = get_object_or_404(Package.objects.normal(), pkgname=name, repo__name__iexact=repo, arch__name=arch) # files are inserted in sorted order, so preserve that fileslist = PackageFile.objects.filter(pkg=pkg).order_by('id') @@ -213,7 +213,7 @@ def files_json(request, name, repo, arch): def download(request, name, repo, arch): - pkg = get_object_or_404(Package, + pkg = get_object_or_404(Package.objects.normal(), pkgname=name, repo__name__iexact=repo, arch__name=arch) url = get_mirror_url_for_download() if not url: diff --git a/packages/views/flag.py b/packages/views/flag.py index dadadd19..edb3f092 100644 --- a/packages/views/flag.py +++ b/packages/views/flag.py @@ -49,7 +49,7 @@ def flaghelp(request): @never_cache def flag(request, name, repo, arch): - pkg = get_object_or_404(Package, + pkg = get_object_or_404(Package.objects.normal(), pkgname=name, repo__name__iexact=repo, arch__name=arch) if pkg.flag_date is not None: # already flagged. do nothing. @@ -158,7 +158,7 @@ def flag_confirmed(request, name, repo, arch): @permission_required('main.change_package') def unflag(request, name, repo, arch): - pkg = get_object_or_404(Package, + pkg = get_object_or_404(Package.objects.normal(), pkgname=name, repo__name__iexact=repo, arch__name=arch) pkg.flag_date = None pkg.save() @@ -166,7 +166,7 @@ def unflag(request, name, repo, arch): @permission_required('main.change_package') def unflag_all(request, name, repo, arch): - pkg = get_object_or_404(Package, + pkg = get_object_or_404(Package.objects.normal(), pkgname=name, repo__name__iexact=repo, arch__name=arch) # find all packages from (hopefully) the same PKGBUILD pkgs = Package.objects.filter(pkgbase=pkg.pkgbase, -- cgit v1.2.3-2-g168b From dc6cc49f6f876983f76f5f8c05a2285801f27ea0 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 23 Jan 2013 09:20:19 -0700 Subject: Use more modern verison of string template formatting Signed-off-by: Dan McGee --- packages/views/display.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'packages') diff --git a/packages/views/display.py b/packages/views/display.py index c2369aba..497c8d48 100644 --- a/packages/views/display.py +++ b/packages/views/display.py @@ -1,6 +1,5 @@ import datetime import json -from string import Template from urllib import urlencode from django.http import HttpResponse, Http404 @@ -223,12 +222,9 @@ def download(request, name, repo, arch): # grab the first non-any arch to fake the download path arch = Arch.objects.exclude(agnostic=True)[0].name values = { - 'host': url.url, - 'arch': arch, - 'repo': pkg.repo.name.lower(), - 'file': pkg.filename, } - url = Template('${host}${repo}/os/${arch}/${file}').substitute(values) + url = '{host}{repo}/os/{arch}/{filename}'.format(host=url.url, + repo=pkg.repo.name.lower(), arch=arch, filename=pkg.filename) return redirect(url) # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b