From 74d2a5df5ca7ee4b6497a6e7609491d72cdbb309 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 3 Nov 2011 17:18:13 -0500 Subject: Refactor more package signoff stuff This sets up some shared utility code for use in a later package signoff email report command. Signed-off-by: Dan McGee --- packages/utils.py | 134 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 125 insertions(+), 9 deletions(-) (limited to 'packages/utils.py') diff --git a/packages/utils.py b/packages/utils.py index c8c1f8a6..42cfbe0f 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -1,12 +1,11 @@ -from collections import defaultdict from operator import itemgetter from django.db import connection from django.db.models import Count, Max -from main.models import Package -from main.utils import cache_function -from .models import PackageGroup, PackageRelation, Signoff +from main.models import Package, Repo +from main.utils import cache_function, groupby_preserve_order, PackageStandin +from .models import PackageGroup, PackageRelation, SignoffSpecification, Signoff @cache_function(300) def get_group_info(include_arches=None): @@ -148,8 +147,90 @@ SELECT DISTINCT id id__in=to_fetch) return relations -def get_current_signoffs(): - '''Returns a mapping of pkgbase -> signoff objects.''' + +DEFAULT_SIGNOFF_SPEC = SignoffSpecification() + +def approved_by_signoffs(signoffs, spec=DEFAULT_SIGNOFF_SPEC): + if signoffs: + good_signoffs = sum(1 for s in signoffs if not s.revoked) + return good_signoffs >= spec.required + return False + +class PackageSignoffGroup(object): + '''Encompasses all packages in testing with the same pkgbase.''' + def __init__(self, packages, user=None): + if len(packages) == 0: + raise Exception + self.packages = packages + self.user = user + self.target_repo = None + self.signoffs = set() + self.specification = DEFAULT_SIGNOFF_SPEC + + first = packages[0] + self.pkgbase = first.pkgbase + self.arch = first.arch + self.repo = first.repo + self.version = '' + self.last_update = first.last_update + self.packager = first.packager + + version = first.full_version + if all(version == pkg.full_version for pkg in packages): + self.version = version + + @property + def package(self): + '''Try and return a relevant single package object representing this + group. Start by seeing if there is only one package, then look for the + matching package by name, finally falling back to a standin package + object.''' + if len(self.packages) == 1: + return self.packages[0] + + same_pkgs = [p for p in self.packages if p.pkgname == p.pkgbase] + if same_pkgs: + return same_pkgs[0] + + return PackageStandin(self.packages[0]) + + def find_signoffs(self, all_signoffs): + '''Look through a list of Signoff objects for ones matching this + particular group and store them on the object.''' + for s in all_signoffs: + if s.pkgbase != self.pkgbase: + continue + if self.version and not s.full_version == self.version: + continue + if s.arch_id == self.arch.id and s.repo_id == self.repo.id: + self.signoffs.add(s) + + def approved(self): + return approved_by_signoffs(self.signoffs, self.specification) + + @property + def completed(self): + return sum(1 for s in self.signoffs if not s.revoked) + + @property + def required(self): + return self.specification.required + + def user_signed_off(self, user=None): + '''Did a given user signoff on this package? user can be passed as an + argument, or attached to the group object itself so this can be called + from a template.''' + if user is None: + user = self.user + return user in (s.user for s in self.signoffs if not s.revoked) + + def __unicode__(self): + return u'%s-%s (%s): %d' % ( + self.pkgbase, self.version, self.arch, len(self.signoffs)) + +def get_current_signoffs(repos): + '''Returns a mapping of pkgbase -> signoff objects for the given repos.''' + cursor = connection.cursor() sql = """ SELECT DISTINCT s.id FROM packages_signoff s @@ -162,14 +243,49 @@ SELECT DISTINCT s.id AND s.repo_id = p.repo_id ) JOIN repos r ON p.repo_id = r.id - WHERE r.testing = %s + WHERE r.id IN ( """ - cursor = connection.cursor() - cursor.execute(sql, [True]) + sql += ", ".join("%s" for r in repos) + sql += ")" + cursor.execute(sql, [r.id for r in repos]) + results = cursor.fetchall() # fetch all of the returned signoffs by ID to_fetch = [row[0] for row in results] signoffs = Signoff.objects.select_related('user').in_bulk(to_fetch) return signoffs.values() +def get_target_repo_map(pkgbases): + package_repos = Package.objects.order_by().values_list( + 'pkgbase', 'repo__name').filter( + repo__testing=False, repo__staging=False, + pkgbase__in=pkgbases).distinct() + return dict(package_repos) + +def get_signoff_groups(repos=None): + if repos is None: + repos = Repo.objects.filter(testing=True) + + test_pkgs = Package.objects.normal().filter(repo__in=repos) + packages = test_pkgs.order_by('pkgname') + + # Collect all pkgbase values in testing repos + q_pkgbase = test_pkgs.values('pkgbase') + pkgtorepo = get_target_repo_map(q_pkgbase) + + # Collect all existing signoffs for these packages + signoffs = get_current_signoffs(repos) + + same_pkgbase_key = lambda x: (x.repo.name, x.arch.name, x.pkgbase) + grouped = groupby_preserve_order(packages, same_pkgbase_key) + signoff_groups = [] + for group in grouped: + signoff_group = PackageSignoffGroup(group) + signoff_group.target_repo = pkgtorepo.get(signoff_group.pkgbase, + "Unknown") + signoff_group.find_signoffs(signoffs) + signoff_groups.append(signoff_group) + + return signoff_groups + # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From 5f2c3bf98baabf919681525e600639643aa2c119 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 3 Nov 2011 20:39:59 -0500 Subject: Signoffs changes and improvements * Better signoff report with more detail * Show signoff specification in signoffs view * Honor disabled/bad flags and display in approval column * Various other small bugfixes and tweaks Signed-off-by: Dan McGee --- packages/utils.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'packages/utils.py') diff --git a/packages/utils.py b/packages/utils.py index 42cfbe0f..60b95e21 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -5,7 +5,8 @@ from django.db.models import Count, Max from main.models import Package, Repo from main.utils import cache_function, groupby_preserve_order, PackageStandin -from .models import PackageGroup, PackageRelation, SignoffSpecification, Signoff +from .models import (PackageGroup, PackageRelation, + SignoffSpecification, Signoff, DEFAULT_SIGNOFF_SPEC) @cache_function(300) def get_group_info(include_arches=None): @@ -148,9 +149,7 @@ SELECT DISTINCT id return relations -DEFAULT_SIGNOFF_SPEC = SignoffSpecification() - -def approved_by_signoffs(signoffs, spec=DEFAULT_SIGNOFF_SPEC): +def approved_by_signoffs(signoffs, spec): if signoffs: good_signoffs = sum(1 for s in signoffs if not s.revoked) return good_signoffs >= spec.required @@ -158,14 +157,13 @@ def approved_by_signoffs(signoffs, spec=DEFAULT_SIGNOFF_SPEC): class PackageSignoffGroup(object): '''Encompasses all packages in testing with the same pkgbase.''' - def __init__(self, packages, user=None): + def __init__(self, packages): if len(packages) == 0: raise Exception self.packages = packages - self.user = user + self.user = None self.target_repo = None self.signoffs = set() - self.specification = DEFAULT_SIGNOFF_SPEC first = packages[0] self.pkgbase = first.pkgbase @@ -175,6 +173,10 @@ class PackageSignoffGroup(object): self.last_update = first.last_update self.packager = first.packager + self.specification = \ + SignoffSpecification.objects.get_or_default_from_package(first) + self.default_spec = self.specification is DEFAULT_SIGNOFF_SPEC + version = first.full_version if all(version == pkg.full_version for pkg in packages): self.version = version -- cgit v1.2.3-2-g168b From 0aa42e2c01df2bf1c9e425994420f5ae10252597 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 3 Nov 2011 21:32:30 -0500 Subject: Allow signoff manipulation if you are a maintainer This is a more expensive and not-yet-optimized way of doing this, but we can fix that later as needed. Signed-off-by: Dan McGee --- packages/utils.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'packages/utils.py') diff --git a/packages/utils.py b/packages/utils.py index 60b95e21..1a2c0de0 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -2,6 +2,7 @@ from operator import itemgetter from django.db import connection from django.db.models import Count, Max +from django.contrib.auth.models import User from main.models import Package, Repo from main.utils import cache_function, groupby_preserve_order, PackageStandin @@ -172,6 +173,9 @@ class PackageSignoffGroup(object): self.version = '' self.last_update = first.last_update self.packager = first.packager + self.maintainers = User.objects.filter( + package_relations__type=PackageRelation.MAINTAINER, + package_relations__pkgbase=self.pkgbase) self.specification = \ SignoffSpecification.objects.get_or_default_from_package(first) -- cgit v1.2.3-2-g168b From 278d74b1d12568d4c9b6d5533e57e820d038ae64 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 3 Nov 2011 22:05:35 -0500 Subject: Minor signoff query tweaks/optimizations Signed-off-by: Dan McGee --- packages/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'packages/utils.py') diff --git a/packages/utils.py b/packages/utils.py index 1a2c0de0..65769baf 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -248,8 +248,7 @@ SELECT DISTINCT s.id AND s.arch_id = p.arch_id AND s.repo_id = p.repo_id ) - JOIN repos r ON p.repo_id = r.id - WHERE r.id IN ( + WHERE p.repo_id IN ( """ sql += ", ".join("%s" for r in repos) sql += ")" @@ -264,15 +263,16 @@ SELECT DISTINCT s.id def get_target_repo_map(pkgbases): package_repos = Package.objects.order_by().values_list( 'pkgbase', 'repo__name').filter( - repo__testing=False, repo__staging=False, pkgbase__in=pkgbases).distinct() return dict(package_repos) def get_signoff_groups(repos=None): if repos is None: repos = Repo.objects.filter(testing=True) + repo_ids = [r.pk for r in repos] - test_pkgs = Package.objects.normal().filter(repo__in=repos) + test_pkgs = Package.objects.select_related( + 'arch', 'repo', 'packager').filter(repo__in=repo_ids) packages = test_pkgs.order_by('pkgname') # Collect all pkgbase values in testing repos -- cgit v1.2.3-2-g168b From 19c2841f20653fd3c59f73fdb16f7f7b1ea15434 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 3 Nov 2011 22:46:36 -0500 Subject: Add new attach_maintainers() utility method This allows us to alleviate the N+1 query problem when we want maintainer data for a queryset of packages. We use it on signoffs here; we should also be able to apply this to the todolist section where this problem has existed for some time. Signed-off-by: Dan McGee --- packages/utils.py | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) (limited to 'packages/utils.py') diff --git a/packages/utils.py b/packages/utils.py index 65769baf..0d756a85 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -1,3 +1,4 @@ +from collections import defaultdict from operator import itemgetter from django.db import connection @@ -127,6 +128,7 @@ SELECT p.id, q.id differences.sort(key=lambda a: (a.repo.name, a.pkgname)) return differences + def get_wrong_permissions(): sql = """ SELECT DISTINCT id @@ -150,6 +152,32 @@ SELECT DISTINCT id return relations +def attach_maintainers(packages): + '''Given a queryset or something resembling it of package objects, find all + the maintainers and attach them to the packages to prevent N+1 query + cascading.''' + pkgbases = set(p.pkgbase for p in packages) + rels = PackageRelation.objects.filter(type=PackageRelation.MAINTAINER, + pkgbase__in=pkgbases).values_list('pkgbase', 'user_id').distinct() + + # get all the user objects we will need + user_ids = set(rel[1] for rel in rels) + users = User.objects.in_bulk(user_ids) + + # now build a pkgbase -> [maintainers...] map + maintainers = defaultdict(list) + for rel in rels: + maintainers[rel[0]].append(users[rel[1]]) + + annotated = [] + # and finally, attach the maintainer lists on the original packages + for package in packages: + package.maintainers = maintainers[package.pkgbase] + annotated.append(package) + + return annotated + + def approved_by_signoffs(signoffs, spec): if signoffs: good_signoffs = sum(1 for s in signoffs if not s.revoked) @@ -173,9 +201,7 @@ class PackageSignoffGroup(object): self.version = '' self.last_update = first.last_update self.packager = first.packager - self.maintainers = User.objects.filter( - package_relations__type=PackageRelation.MAINTAINER, - package_relations__pkgbase=self.pkgbase) + self.maintainers = first.maintainers self.specification = \ SignoffSpecification.objects.get_or_default_from_package(first) @@ -236,6 +262,7 @@ class PackageSignoffGroup(object): def get_current_signoffs(repos): '''Returns a mapping of pkgbase -> signoff objects for the given repos.''' + # TODO this isn't current at all- this is every single signoff... cursor = connection.cursor() sql = """ SELECT DISTINCT s.id @@ -274,6 +301,7 @@ def get_signoff_groups(repos=None): test_pkgs = Package.objects.select_related( 'arch', 'repo', 'packager').filter(repo__in=repo_ids) packages = test_pkgs.order_by('pkgname') + packages = attach_maintainers(packages) # Collect all pkgbase values in testing repos q_pkgbase = test_pkgs.values('pkgbase') -- cgit v1.2.3-2-g168b From 0db2830b8fda4d898a184a31f3375c10f3cc4083 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 3 Nov 2011 23:30:16 -0500 Subject: Make maintainer lookup on todo lists fast This is rather sick to look at. Sorry, Django gives me no other choice. Signed-off-by: Dan McGee --- packages/utils.py | 1 + 1 file changed, 1 insertion(+) (limited to 'packages/utils.py') diff --git a/packages/utils.py b/packages/utils.py index 0d756a85..4af0f67d 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -156,6 +156,7 @@ def attach_maintainers(packages): '''Given a queryset or something resembling it of package objects, find all the maintainers and attach them to the packages to prevent N+1 query cascading.''' + packages = list(packages) pkgbases = set(p.pkgbase for p in packages) rels = PackageRelation.objects.filter(type=PackageRelation.MAINTAINER, pkgbase__in=pkgbases).values_list('pkgbase', 'user_id').distinct() -- cgit v1.2.3-2-g168b From 94f46acebf03652d7ad2ed504d4ce863d5cbd913 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 4 Nov 2011 00:01:51 -0500 Subject: Find all potential package signoff specifications upfront This should save a significant amount of time in the case where there are a lot of signups to look up; at least one query per signoff row. Signed-off-by: Dan McGee --- packages/utils.py | 54 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 15 deletions(-) (limited to 'packages/utils.py') diff --git a/packages/utils.py b/packages/utils.py index 4af0f67d..5240ae23 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -194,6 +194,8 @@ class PackageSignoffGroup(object): self.user = None self.target_repo = None self.signoffs = set() + self.specification = DEFAULT_SIGNOFF_SPEC + self.default_spec = True first = packages[0] self.pkgbase = first.pkgbase @@ -204,10 +206,6 @@ class PackageSignoffGroup(object): self.packager = first.packager self.maintainers = first.maintainers - self.specification = \ - SignoffSpecification.objects.get_or_default_from_package(first) - self.default_spec = self.specification is DEFAULT_SIGNOFF_SPEC - version = first.full_version if all(version == pkg.full_version for pkg in packages): self.version = version @@ -238,6 +236,17 @@ class PackageSignoffGroup(object): if s.arch_id == self.arch.id and s.repo_id == self.repo.id: self.signoffs.add(s) + def find_specification(self, specifications): + for spec in specifications: + if spec.pkgbase != self.pkgbase: + continue + if self.version and not spec.full_version == self.version: + continue + if spec.arch_id == self.arch.id and spec.repo_id == self.repo.id: + self.specification = spec + self.default_spec = False + return + def approved(self): return approved_by_signoffs(self.signoffs, self.specification) @@ -261,13 +270,9 @@ class PackageSignoffGroup(object): return u'%s-%s (%s): %d' % ( self.pkgbase, self.version, self.arch, len(self.signoffs)) -def get_current_signoffs(repos): - '''Returns a mapping of pkgbase -> signoff objects for the given repos.''' - # TODO this isn't current at all- this is every single signoff... - cursor = connection.cursor() - sql = """ +_SQL_SPEC_OR_SIGNOFF = """ SELECT DISTINCT s.id - FROM packages_signoff s + FROM %s s JOIN packages p ON ( s.pkgbase = p.pkgbase AND s.pkgver = p.pkgver @@ -276,11 +281,16 @@ SELECT DISTINCT s.id AND s.arch_id = p.arch_id AND s.repo_id = p.repo_id ) - WHERE p.repo_id IN ( + AND p.repo_id IN (%s) """ - sql += ", ".join("%s" for r in repos) - sql += ")" - cursor.execute(sql, [r.id for r in repos]) + +def get_current_signoffs(repos): + '''Returns a mapping of pkgbase -> signoff objects for the given repos.''' + cursor = connection.cursor() + # query pre-process- fill in table name and placeholders for IN + sql = _SQL_SPEC_OR_SIGNOFF % ('packages_signoff', + ','.join(['%s' for r in repos])) + cursor.execute(sql, [r.pk for r in repos]) results = cursor.fetchall() # fetch all of the returned signoffs by ID @@ -288,6 +298,18 @@ SELECT DISTINCT s.id signoffs = Signoff.objects.select_related('user').in_bulk(to_fetch) return signoffs.values() +def get_current_specifications(repos): + '''Returns a mapping of pkgbase -> signoff specification objects for the + given repos.''' + cursor = connection.cursor() + sql = _SQL_SPEC_OR_SIGNOFF % ('packages_signoffspecification', + ','.join(['%s' for r in repos])) + cursor.execute(sql, [r.pk for r in repos]) + + results = cursor.fetchall() + to_fetch = [row[0] for row in results] + return SignoffSpecification.objects.in_bulk(to_fetch).values() + def get_target_repo_map(pkgbases): package_repos = Package.objects.order_by().values_list( 'pkgbase', 'repo__name').filter( @@ -308,8 +330,9 @@ def get_signoff_groups(repos=None): q_pkgbase = test_pkgs.values('pkgbase') pkgtorepo = get_target_repo_map(q_pkgbase) - # Collect all existing signoffs for these packages + # Collect all possible signoffs and specifications for these packages signoffs = get_current_signoffs(repos) + specs = get_current_specifications(repos) same_pkgbase_key = lambda x: (x.repo.name, x.arch.name, x.pkgbase) grouped = groupby_preserve_order(packages, same_pkgbase_key) @@ -319,6 +342,7 @@ def get_signoff_groups(repos=None): signoff_group.target_repo = pkgtorepo.get(signoff_group.pkgbase, "Unknown") signoff_group.find_signoffs(signoffs) + signoff_group.find_specification(specs) signoff_groups.append(signoff_group) return signoff_groups -- cgit v1.2.3-2-g168b From 20ebe658921c2ce78bf9a05116de045ee38f0820 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 4 Nov 2011 10:43:02 -0500 Subject: Fix signoff target repo mapping I clearly should not have removed this code yesterday, otherwise packages have their target repo matched to a testing one. Signed-off-by: Dan McGee --- packages/utils.py | 1 + 1 file changed, 1 insertion(+) (limited to 'packages/utils.py') diff --git a/packages/utils.py b/packages/utils.py index 5240ae23..ddd822e4 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -313,6 +313,7 @@ def get_current_specifications(repos): def get_target_repo_map(pkgbases): package_repos = Package.objects.order_by().values_list( 'pkgbase', 'repo__name').filter( + repo__testing=False, repo__staging=False, pkgbase__in=pkgbases).distinct() return dict(package_repos) -- cgit v1.2.3-2-g168b From 28f72db7be7bf2f54d734c78422e6179f0ce29f1 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 4 Nov 2011 12:38:57 -0500 Subject: Rewrite get_target_repo_map() using raw SQL This improves the shitty query plan brought upon us by MySQL by rewriting it to use JOINs only and no dependent subqueries. Signed-off-by: Dan McGee --- packages/utils.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) (limited to 'packages/utils.py') diff --git a/packages/utils.py b/packages/utils.py index ddd822e4..b21ac557 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -310,12 +310,25 @@ def get_current_specifications(repos): to_fetch = [row[0] for row in results] return SignoffSpecification.objects.in_bulk(to_fetch).values() -def get_target_repo_map(pkgbases): - package_repos = Package.objects.order_by().values_list( - 'pkgbase', 'repo__name').filter( - repo__testing=False, repo__staging=False, - pkgbase__in=pkgbases).distinct() - return dict(package_repos) +def get_target_repo_map(repos): + sql = """ +SELECT DISTINCT p1.pkgbase, r.name + FROM packages p1 + JOIN repos r ON p1.repo_id = r.id + JOIN packages p2 ON p1.pkgbase = p2.pkgbase + WHERE r.staging = %s + AND r.testing = %s + AND p2.repo_id IN ( + """ + sql += ','.join(['%s' for r in repos]) + sql += ")" + + params = [False, False] + params.extend(r.pk for r in repos) + + cursor = connection.cursor() + cursor.execute(sql, params) + return dict(cursor.fetchall()) def get_signoff_groups(repos=None): if repos is None: @@ -328,8 +341,7 @@ def get_signoff_groups(repos=None): packages = attach_maintainers(packages) # Collect all pkgbase values in testing repos - q_pkgbase = test_pkgs.values('pkgbase') - pkgtorepo = get_target_repo_map(q_pkgbase) + pkgtorepo = get_target_repo_map(repos) # Collect all possible signoffs and specifications for these packages signoffs = get_current_signoffs(repos) -- cgit v1.2.3-2-g168b From 022692b3f33de8c45741d3cb27fa95f9f6facdea Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 11 Nov 2011 10:43:18 -0600 Subject: Show relevant signoffs on dashboard Signed-off-by: Dan McGee --- packages/utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'packages/utils.py') diff --git a/packages/utils.py b/packages/utils.py index b21ac557..0df0e382 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -330,7 +330,7 @@ SELECT DISTINCT p1.pkgbase, r.name cursor.execute(sql, params) return dict(cursor.fetchall()) -def get_signoff_groups(repos=None): +def get_signoff_groups(repos=None, user=None): if repos is None: repos = Repo.objects.filter(testing=True) repo_ids = [r.pk for r in repos] @@ -340,6 +340,11 @@ def get_signoff_groups(repos=None): packages = test_pkgs.order_by('pkgname') packages = attach_maintainers(packages) + # Filter by user if asked to do so + if user is not None: + packages = [p for p in packages if user == p.packager + or user in p.maintainers] + # Collect all pkgbase values in testing repos pkgtorepo = get_target_repo_map(repos) -- cgit v1.2.3-2-g168b From a883b0af23143364ab0724fda2ecdef9aba8191f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 11 Nov 2011 11:57:04 -0600 Subject: Add a split packages sitemap With very low priority, but this should at least give a few more cross-linking pages to any crawlers using sitemaps. Signed-off-by: Dan McGee --- packages/utils.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) (limited to 'packages/utils.py') diff --git a/packages/utils.py b/packages/utils.py index 0df0e382..f8e1f2a1 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -2,10 +2,10 @@ from collections import defaultdict from operator import itemgetter from django.db import connection -from django.db.models import Count, Max +from django.db.models import Count, Max, F from django.contrib.auth.models import User -from main.models import Package, Repo +from main.models import Package, Arch, Repo from main.utils import cache_function, groupby_preserve_order, PackageStandin from .models import (PackageGroup, PackageRelation, SignoffSpecification, Signoff, DEFAULT_SIGNOFF_SPEC) @@ -49,6 +49,20 @@ def get_group_info(include_arches=None): groups.extend(val.itervalues()) return sorted(groups, key=itemgetter('name', 'arch')) +def get_split_packages_info(): + '''Return info on split packages that do not have an actual package name + matching the split pkgbase.''' + pkgnames = Package.objects.values('pkgname') + split_pkgs = Package.objects.exclude(pkgname=F('pkgbase')).exclude( + pkgbase__in=pkgnames).values('pkgbase', 'repo', 'arch').annotate( + last_update=Max('last_update')) + all_arches = Arch.objects.in_bulk(set(s['arch'] for s in split_pkgs)) + all_repos = Repo.objects.in_bulk(set(s['repo'] for s in split_pkgs)) + for split in split_pkgs: + split['arch'] = all_arches[split['arch']] + split['repo'] = all_repos[split['repo']] + return split_pkgs + class Difference(object): def __init__(self, pkgname, repo, pkg_a, pkg_b): self.pkgname = pkgname -- cgit v1.2.3-2-g168b From f71f3e23eda8e1ff447340812025ab724aff14cb Mon Sep 17 00:00:00 2001 From: Johannes Krampf Date: Sat, 3 Dec 2011 14:18:07 +0100 Subject: Fix differences query if there are more than 2 architectures --- packages/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'packages/utils.py') diff --git a/packages/utils.py b/packages/utils.py index f8e1f2a1..82d47bc7 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -106,7 +106,11 @@ SELECT p.id, q.id AND p.arch_id != q.arch_id AND p.id != q.id ) - WHERE p.arch_id IN (%s, %s) + WHERE p.arch_id in (%s, %s) + AND ( + q.arch_id in (%s, %s) + OR q.id IS NULL + ) AND ( q.id IS NULL OR p.pkgver != q.pkgver @@ -115,7 +119,7 @@ SELECT p.id, q.id ) """ cursor = connection.cursor() - cursor.execute(sql, [arch_a.id, arch_b.id]) + cursor.execute(sql, [arch_a.id, arch_b.id, arch_a.id, arch_b.id]) results = cursor.fetchall() # column A will always have a value, column B might be NULL to_fetch = [row[0] for row in results] -- cgit v1.2.3-2-g168b