From 4e1e28729f97eb694cdcae2f3fe51b5178963069 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 27 Apr 2012 09:07:26 -0500 Subject: Use GenericIPAddressField in flag request ip_address New (and slightly odd with regards to verbose_name) in Django 1.4. This simply ensures a deployment in an IPv6 environment actually works as expected. If you were using PostgreSQL as a database backend, you won't be affected by this as the 'inet' type was already used, but at least now you can edit the values in the admin without getting an error. Signed-off-by: Dan McGee --- packages/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index 820e61ba..7a7a81cd 100644 --- a/packages/models.py +++ b/packages/models.py @@ -173,7 +173,9 @@ class FlagRequest(models.Model): user = models.ForeignKey(User, blank=True, null=True) user_email = models.EmailField('email address') created = models.DateTimeField(editable=False) - ip_address = models.IPAddressField('IP address') + # Great work, Django... https://code.djangoproject.com/ticket/18212 + ip_address = models.GenericIPAddressField(verbose_name='IP address', + unpack_ipv4=True) pkgbase = models.CharField(max_length=255, db_index=True) version = models.CharField(max_length=255, default='') repo = models.ForeignKey(Repo) -- cgit v1.2.3-2-g168b From 17e33f9118e9749b1e3fdfd76686e85dbcecfb00 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 18 May 2012 19:17:30 -0500 Subject: Refactor an abstract base class out of conflicts/provides/replaces Signed-off-by: Dan McGee --- packages/models.py | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index 7a7a81cd..f57c9f3c 100644 --- a/packages/models.py +++ b/packages/models.py @@ -218,10 +218,10 @@ class License(models.Model): class Meta: ordering = ['name'] -class Conflict(models.Model): - pkg = models.ForeignKey('main.Package', related_name='conflicts') + +class RelatedToBase(models.Model): + '''A base class for conflicts/provides/replaces/etc.''' name = models.CharField(max_length=255, db_index=True) - comparison = models.CharField(max_length=255, default='') version = models.CharField(max_length=255, default='') def __unicode__(self): @@ -230,36 +230,29 @@ class Conflict(models.Model): return self.name class Meta: + abstract = True ordering = ['name'] -class Provision(models.Model): + +class Conflict(RelatedToBase): + pkg = models.ForeignKey('main.Package', related_name='conflicts') + comparison = models.CharField(max_length=255, default='') + + +class Provision(RelatedToBase): pkg = models.ForeignKey('main.Package', related_name='provides') - name = models.CharField(max_length=255, db_index=True) # comparison must be '=' for provides - comparison = '=' - version = models.CharField(max_length=255, default='') - def __unicode__(self): - if self.version: - return u'%s=%s' % (self.name, self.version) - return self.name + @property + def comparison(self): + if self.version is not None and self.version != '': + return '=' + return None - class Meta: - ordering = ['name'] -class Replacement(models.Model): +class Replacement(RelatedToBase): pkg = models.ForeignKey('main.Package', related_name='replaces') - name = models.CharField(max_length=255, db_index=True) comparison = models.CharField(max_length=255, default='') - version = models.CharField(max_length=255, default='') - - def __unicode__(self): - if self.version: - return u'%s%s%s' % (self.name, self.comparison, self.version) - return self.name - - class Meta: - ordering = ['name'] # hook up some signals -- cgit v1.2.3-2-g168b From 158be107e4ad682de0c9360658dfa5a72c21ee58 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 18 May 2012 19:57:12 -0500 Subject: Add a get_best_satisfier method to RelatedToBase This is basically what we do in PackageDepend already. Signed-off-by: Dan McGee --- packages/models.py | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index f57c9f3c..a4d2a556 100644 --- a/packages/models.py +++ b/packages/models.py @@ -4,7 +4,7 @@ from django.db import models from django.db.models.signals import pre_save from django.contrib.auth.models import User -from main.models import Arch, Repo +from main.models import Arch, Repo, Package from main.utils import set_created_field class PackageRelation(models.Model): @@ -224,6 +224,40 @@ class RelatedToBase(models.Model): name = models.CharField(max_length=255, db_index=True) version = models.CharField(max_length=255, default='') + def get_best_satisfier(self): + '''Find a satisfier for this related package that best matches the + given criteria. It will not search provisions, but will find packages + named and matching repo characteristics if possible.''' + # NOTE: this is cribbed directly from the PackageDepend method of the + # same name. Really, all of these things could use the same method if + # the PackageDepend class was moved here and field names were changed + # to match the layout we use here. + pkgs = Package.objects.normal().filter(pkgname=self.name) + if not self.pkg.arch.agnostic: + # make sure we match architectures if possible + arches = self.pkg.applicable_arches() + pkgs = pkgs.filter(arch__in=arches) + if len(pkgs) == 0: + # couldn't find a package in the DB + # it should be a virtual depend (or a removed package) + return None + if len(pkgs) == 1: + return pkgs[0] + # more than one package, see if we can't shrink it down + # grab the first though in case we fail + pkg = pkgs[0] + # prevents yet more DB queries, these lists should be short; + # after each grab the best available in case we remove all entries + pkgs = [p for p in pkgs if p.repo.staging == self.pkg.repo.staging] + if len(pkgs) > 0: + pkg = pkgs[0] + + pkgs = [p for p in pkgs if p.repo.testing == self.pkg.repo.testing] + if len(pkgs) > 0: + pkg = pkgs[0] + + return pkg + def __unicode__(self): if self.version: return u'%s%s%s' % (self.name, self.comparison, self.version) -- cgit v1.2.3-2-g168b From ef561bcd705e1047b1a81364ac143ce52c60defa Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 18 May 2012 20:54:39 -0500 Subject: Add new depends model Signed-off-by: Dan McGee --- packages/models.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index a4d2a556..c7b1cab4 100644 --- a/packages/models.py +++ b/packages/models.py @@ -268,6 +268,13 @@ class RelatedToBase(models.Model): ordering = ['name'] +class Depend(RelatedToBase): + pkg = models.ForeignKey('main.Package', related_name='depends_new') + comparison = models.CharField(max_length=255, default='') + optional = models.BooleanField(default=False) + description = models.TextField(null=True, blank=True) + + class Conflict(RelatedToBase): pkg = models.ForeignKey('main.Package', related_name='conflicts') comparison = models.CharField(max_length=255, default='') -- cgit v1.2.3-2-g168b From 72a92102df4999dbcc370064707c9026d51c4fe7 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 18 May 2012 21:29:03 -0500 Subject: Switch to usage of new Depend object Signed-off-by: Dan McGee --- packages/models.py | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index c7b1cab4..cb65f1f1 100644 --- a/packages/models.py +++ b/packages/models.py @@ -228,10 +228,6 @@ class RelatedToBase(models.Model): '''Find a satisfier for this related package that best matches the given criteria. It will not search provisions, but will find packages named and matching repo characteristics if possible.''' - # NOTE: this is cribbed directly from the PackageDepend method of the - # same name. Really, all of these things could use the same method if - # the PackageDepend class was moved here and field names were changed - # to match the layout we use here. pkgs = Package.objects.normal().filter(pkgname=self.name) if not self.pkg.arch.agnostic: # make sure we match architectures if possible @@ -258,6 +254,36 @@ class RelatedToBase(models.Model): return pkg + def get_providers(self): + '''Return providers of this related package. Does *not* include exact + matches as it checks the Provision names only, use get_best_satisfier() + instead for exact matches.''' + pkgs = Package.objects.normal().filter( + provides__name=self.name).order_by().distinct() + if not self.pkg.arch.agnostic: + # make sure we match architectures if possible + arches = self.pkg.applicable_arches() + pkgs = pkgs.filter(arch__in=arches) + + # Logic here is to filter out packages that are in multiple repos if + # they are not requested. For example, if testing is False, only show a + # testing package if it doesn't exist in a non-testing repo. + filtered = {} + for package in pkgs: + if package.pkgname not in filtered or \ + package.repo.staging == self.pkg.repo.staging: + filtered[package.pkgname] = package + pkgs = filtered.values() + + filtered = {} + for package in pkgs: + if package.pkgname not in filtered or \ + package.repo.testing == self.pkg.repo.testing: + filtered[package.pkgname] = package + pkgs = filtered.values() + + return pkgs + def __unicode__(self): if self.version: return u'%s%s%s' % (self.name, self.comparison, self.version) -- cgit v1.2.3-2-g168b From e1f9a3c44a1449a36f3981b868814f3d1f65d70d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 18 May 2012 21:35:28 -0500 Subject: Drop old PackageDepend model Signed-off-by: Dan McGee --- packages/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index cb65f1f1..c3a16fc5 100644 --- a/packages/models.py +++ b/packages/models.py @@ -295,7 +295,7 @@ class RelatedToBase(models.Model): class Depend(RelatedToBase): - pkg = models.ForeignKey('main.Package', related_name='depends_new') + pkg = models.ForeignKey('main.Package', related_name='depends') comparison = models.CharField(max_length=255, default='') optional = models.BooleanField(default=False) description = models.TextField(null=True, blank=True) -- cgit v1.2.3-2-g168b From b547d41dbf338fb75eb2c6ae05da143a5cd32c74 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 1 Jul 2012 19:40:42 -0500 Subject: Remove no-longer necessary delayed imports of Package Since commit 158be107e4ad6, we have been importing the Package model at the top-level in this file, so we can kill this code that was never updated. This should also give us back any performance hit we were seeing from the delayed imports. Signed-off-by: Dan McGee --- packages/models.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index c3a16fc5..2c6ad43c 100644 --- a/packages/models.py +++ b/packages/models.py @@ -26,8 +26,6 @@ class PackageRelation(models.Model): created = models.DateTimeField(editable=False) def get_associated_packages(self): - # TODO: delayed import to avoid circular reference - from main.models import Package return Package.objects.normal().filter(pkgbase=self.pkgbase) def repositories(self): @@ -146,8 +144,6 @@ class Signoff(models.Model): @property def packages(self): - # TODO: delayed import to avoid circular reference - from main.models import Package return Package.objects.normal().filter(pkgbase=self.pkgbase, pkgver=self.pkgver, pkgrel=self.pkgrel, epoch=self.epoch, arch=self.arch, repo=self.repo) @@ -202,14 +198,15 @@ class PackageGroup(models.Model): Represents a group a package is in. There is no actual group entity, only names that link to given packages. ''' - pkg = models.ForeignKey('main.Package', related_name='groups') + pkg = models.ForeignKey(Package, related_name='groups') name = models.CharField(max_length=255, db_index=True) def __unicode__(self): return "%s: %s" % (self.name, self.pkg) + class License(models.Model): - pkg = models.ForeignKey('main.Package', related_name='licenses') + pkg = models.ForeignKey(Package, related_name='licenses') name = models.CharField(max_length=255) def __unicode__(self): @@ -295,19 +292,19 @@ class RelatedToBase(models.Model): class Depend(RelatedToBase): - pkg = models.ForeignKey('main.Package', related_name='depends') + pkg = models.ForeignKey(Package, related_name='depends') comparison = models.CharField(max_length=255, default='') optional = models.BooleanField(default=False) description = models.TextField(null=True, blank=True) class Conflict(RelatedToBase): - pkg = models.ForeignKey('main.Package', related_name='conflicts') + pkg = models.ForeignKey(Package, related_name='conflicts') comparison = models.CharField(max_length=255, default='') class Provision(RelatedToBase): - pkg = models.ForeignKey('main.Package', related_name='provides') + pkg = models.ForeignKey(Package, related_name='provides') # comparison must be '=' for provides @property @@ -318,7 +315,7 @@ class Provision(RelatedToBase): class Replacement(RelatedToBase): - pkg = models.ForeignKey('main.Package', related_name='replaces') + pkg = models.ForeignKey(Package, related_name='replaces') comparison = models.CharField(max_length=255, default='') -- cgit v1.2.3-2-g168b From 43b5c29b3d89cc2e7e7109bb3c7717a87cfc67b5 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 1 Jul 2012 19:57:16 -0500 Subject: Add new package Update model This will be used to track updates to package as we do them during reporead. By storing enough relevant fields from the package object, we should be able to produce a useful report on a regular basis of what has been happening in the repositories. Signed-off-by: Dan McGee --- packages/models.py | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index 2c6ad43c..5ee06575 100644 --- a/packages/models.py +++ b/packages/models.py @@ -2,11 +2,13 @@ from collections import namedtuple from django.db import models from django.db.models.signals import pre_save +from django.contrib.admin.models import ADDITION, CHANGE, DELETION from django.contrib.auth.models import User from main.models import Arch, Repo, Package from main.utils import set_created_field + class PackageRelation(models.Model): ''' Represents maintainership (or interest) in a package by a given developer. @@ -193,6 +195,66 @@ class FlagRequest(models.Model): def __unicode__(self): return u'%s from %s on %s' % (self.pkgbase, self.who(), self.created) + +UPDATE_ACTION_CHOICES = ( + (ADDITION, 'Addition'), + (CHANGE, 'Change'), + (DELETION, 'Deletion'), +) + + +class Update(models.Model): + package = models.ForeignKey(Package, related_name="updates", + null=True, on_delete=models.SET_NULL) + repo = models.ForeignKey(Repo, related_name="updates") + arch = models.ForeignKey(Arch, related_name="updates") + pkgname = models.CharField(max_length=255) + pkgbase = models.CharField(max_length=255) + action_flag = models.PositiveSmallIntegerField('action flag', + choices=UPDATE_ACTION_CHOICES) + created = models.DateTimeField(editable=False) + + old_pkgver = models.CharField(max_length=255, null=True) + old_pkgrel = models.CharField(max_length=255, null=True) + old_epoch = models.PositiveIntegerField(null=True) + + new_pkgver = models.CharField(max_length=255, null=True) + new_pkgrel = models.CharField(max_length=255, null=True) + new_epoch = models.PositiveIntegerField(null=True) + + class Meta: + get_latest_by = 'created' + + def is_addition(self): + return self.action_flag == ADDITION + + def is_change(self): + return self.action_flag == CHANGE + + def is_deletion(self): + return self.action_flag == DELETION + + @property + def old_version(self): + if self.action_flag == ADDITION: + return None + if self.old_epoch > 0: + return u'%d:%s-%s' % (self.old_epoch, self.old_pkgver, self.old_pkgrel) + return u'%s-%s' % (self.old_pkgver, self.old_pkgrel) + + @property + def new_version(self): + if self.action_flag == DELETION: + return None + if self.new_epoch > 0: + return u'%d:%s-%s' % (self.new_epoch, self.new_pkgver, self.new_pkgrel) + return u'%s-%s' % (self.new_pkgver, self.new_pkgrel) + + def __unicode__(self): + return u'%s of %s on %s' % (self.get_action_flag_display(), + self.pkgname, self.created) + + class PackageGroup(models.Model): ''' Represents a group a package is in. There is no actual group entity, @@ -320,7 +382,7 @@ class Replacement(RelatedToBase): # hook up some signals -for sender in (PackageRelation, SignoffSpecification, Signoff): +for sender in (PackageRelation, SignoffSpecification, Signoff, Update): pre_save.connect(set_created_field, sender=sender, dispatch_uid="packages.models") -- cgit v1.2.3-2-g168b From a87fe016d1a1bf7fdcd2b19f515aa72a5b93db2b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 1 Jul 2012 20:21:34 -0500 Subject: Log package updates during reporead invocation This adds a Manager and log_update method to help log all updates made to the packages table during reporead runs. Signed-off-by: Dan McGee --- packages/models.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index 5ee06575..04f35f9d 100644 --- a/packages/models.py +++ b/packages/models.py @@ -203,6 +203,40 @@ UPDATE_ACTION_CHOICES = ( ) +class UpdateManager(models.Manager): + def log_update(self, old_pkg, new_pkg): + '''Utility method to help log an update. This will determine the type + based on how many packages are passed in, and will pull the relevant + necesary fields off the given packages.''' + update = Update() + if new_pkg: + update.action_flag = ADDITION + update.package = new_pkg + update.arch = new_pkg.arch + update.repo = new_pkg.repo + update.pkgname = new_pkg.pkgname + update.pkgbase = new_pkg.pkgbase + update.new_pkgver = new_pkg.pkgver + update.new_pkgrel = new_pkg.pkgrel + update.new_epoch = new_pkg.epoch + if old_pkg: + if new_pkg: + update.action_flag = CHANGE + else: + update.action_flag = DELETION + update.arch = old_pkg.arch + update.repo = old_pkg.repo + update.pkgname = old_pkg.pkgname + update.pkgbase = old_pkg.pkgbase + + update.old_pkgver = old_pkg.pkgver + update.old_pkgrel = old_pkg.pkgrel + update.old_epoch = old_pkg.epoch + + update.save(force_insert=True) + return update + + class Update(models.Model): package = models.ForeignKey(Package, related_name="updates", null=True, on_delete=models.SET_NULL) @@ -222,6 +256,8 @@ class Update(models.Model): new_pkgrel = models.CharField(max_length=255, null=True) new_epoch = models.PositiveIntegerField(null=True) + objects = UpdateManager() + class Meta: get_latest_by = 'created' -- cgit v1.2.3-2-g168b From 34e877d3328edd93b24bc82e63a89781d4350658 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 1 Jul 2012 20:35:34 -0500 Subject: Add indexes on 'created' field to several package-related models These models regularly sort by or limit by the created field, so adding a index on the created database column makes sense. Signed-off-by: Dan McGee --- packages/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index 04f35f9d..2f03a28b 100644 --- a/packages/models.py +++ b/packages/models.py @@ -138,7 +138,7 @@ class Signoff(models.Model): arch = models.ForeignKey(Arch) repo = models.ForeignKey(Repo) user = models.ForeignKey(User, related_name="package_signoffs") - created = models.DateTimeField(editable=False) + created = models.DateTimeField(editable=False, db_index=True) revoked = models.DateTimeField(null=True) comments = models.TextField(null=True, blank=True) @@ -170,7 +170,7 @@ class FlagRequest(models.Model): ''' user = models.ForeignKey(User, blank=True, null=True) user_email = models.EmailField('email address') - created = models.DateTimeField(editable=False) + created = models.DateTimeField(editable=False, db_index=True) # Great work, Django... https://code.djangoproject.com/ticket/18212 ip_address = models.GenericIPAddressField(verbose_name='IP address', unpack_ipv4=True) @@ -246,7 +246,7 @@ class Update(models.Model): pkgbase = models.CharField(max_length=255) action_flag = models.PositiveSmallIntegerField('action flag', choices=UPDATE_ACTION_CHOICES) - created = models.DateTimeField(editable=False) + created = models.DateTimeField(editable=False, db_index=True) old_pkgver = models.CharField(max_length=255, null=True) old_pkgrel = models.CharField(max_length=255, null=True) -- cgit v1.2.3-2-g168b From 26a00cadcebc0b37775954d261ec73f927ceca12 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 7 Jul 2012 17:28:02 -0500 Subject: Don't log package updates in Python when we have DB trigger support This adds a helper method to find the database engine in use, and then skips code we shouldn't execute if we are doing this another way. Note that this helper method could be useful for backend-specific code paths elsewhere, such as custom SQL being called or lack of StdDev() in sqlite3 out of the box. Signed-off-by: Dan McGee --- packages/models.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index 2f03a28b..45ff3c08 100644 --- a/packages/models.py +++ b/packages/models.py @@ -6,7 +6,7 @@ from django.contrib.admin.models import ADDITION, CHANGE, DELETION from django.contrib.auth.models import User from main.models import Arch, Repo, Package -from main.utils import set_created_field +from main.utils import set_created_field, database_vendor class PackageRelation(models.Model): @@ -207,7 +207,12 @@ class UpdateManager(models.Manager): def log_update(self, old_pkg, new_pkg): '''Utility method to help log an update. This will determine the type based on how many packages are passed in, and will pull the relevant - necesary fields off the given packages.''' + necesary fields off the given packages. + Note that in some cases, this is a no-op if we know this database type + supports triggers to add these rows instead.''' + if database_vendor(Package, 'write') in ('sqlite', 'postgresql'): + # we log updates using database triggers for these backends + return update = Update() if new_pkg: update.action_flag = ADDITION @@ -222,6 +227,12 @@ class UpdateManager(models.Manager): if old_pkg: if new_pkg: update.action_flag = CHANGE + # ensure we should even be logging this + if (old_pkg.pkgver == new_pkg.pkgver and + old_pkg.pkgrel == new_pkg.pkgrel and + old_pkg.epoch == new_pkg.epoch): + # all relevant fields were the same; e.g. a force update + return else: update.action_flag = DELETION update.arch = old_pkg.arch -- cgit v1.2.3-2-g168b From 946d90d08f29094153142056a1778cd595e568a3 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 23 Jul 2012 09:45:44 -0500 Subject: Add '410 Gone' support for packages moved out of repositories This allows us to do better than a generic 404 handler when we know a package previously existed in a given repository, and should also make things a bit nicer when getting sent in from a search engine to a page that no longer exists. Signed-off-by: Dan McGee --- packages/models.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index 45ff3c08..18a62a3c 100644 --- a/packages/models.py +++ b/packages/models.py @@ -297,6 +297,9 @@ class Update(models.Model): return u'%d:%s-%s' % (self.new_epoch, self.new_pkgver, self.new_pkgrel) return u'%s-%s' % (self.new_pkgver, self.new_pkgrel) + def elsewhere(self): + return Package.objects.filter(pkgname=self.pkgname, arch=self.arch) + def __unicode__(self): return u'%s of %s on %s' % (self.get_action_flag_display(), self.pkgname, self.created) -- cgit v1.2.3-2-g168b From 61b4098c611592d62b40a9ee941976a869dff4fc Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 24 Jul 2012 08:55:48 -0500 Subject: Add index on package updates pkgname field Signed-off-by: Dan McGee --- packages/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index 18a62a3c..5b48b30f 100644 --- a/packages/models.py +++ b/packages/models.py @@ -253,7 +253,7 @@ class Update(models.Model): null=True, on_delete=models.SET_NULL) repo = models.ForeignKey(Repo, related_name="updates") arch = models.ForeignKey(Arch, related_name="updates") - pkgname = models.CharField(max_length=255) + pkgname = models.CharField(max_length=255, db_index=True) pkgbase = models.CharField(max_length=255) action_flag = models.PositiveSmallIntegerField('action flag', choices=UPDATE_ACTION_CHOICES) -- cgit v1.2.3-2-g168b From 686942b8788fa43031b3999ac00d60baadc82f53 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 31 Jul 2012 19:56:49 -0500 Subject: Declare 'enums' at class scope Signed-off-by: Dan McGee --- packages/models.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index 5b48b30f..1d538cce 100644 --- a/packages/models.py +++ b/packages/models.py @@ -196,13 +196,6 @@ class FlagRequest(models.Model): return u'%s from %s on %s' % (self.pkgbase, self.who(), self.created) -UPDATE_ACTION_CHOICES = ( - (ADDITION, 'Addition'), - (CHANGE, 'Change'), - (DELETION, 'Deletion'), -) - - class UpdateManager(models.Manager): def log_update(self, old_pkg, new_pkg): '''Utility method to help log an update. This will determine the type @@ -249,6 +242,12 @@ class UpdateManager(models.Manager): class Update(models.Model): + UPDATE_ACTION_CHOICES = ( + (ADDITION, 'Addition'), + (CHANGE, 'Change'), + (DELETION, 'Deletion'), + ) + package = models.ForeignKey(Package, related_name="updates", null=True, on_delete=models.SET_NULL) repo = models.ForeignKey(Repo, related_name="updates") -- cgit v1.2.3-2-g168b From 566a9803dd4928fa2145ef14da2d59d2631eeb05 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 31 Jul 2012 20:07:15 -0500 Subject: Add new deptype column to package depends This is more flexible than our existing 'optional' boolean and will allow us to import check and make depends into the database as well as what we are already doing. Signed-off-by: Dan McGee --- packages/models.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index 1d538cce..b3752b6c 100644 --- a/packages/models.py +++ b/packages/models.py @@ -403,10 +403,19 @@ class RelatedToBase(models.Model): class Depend(RelatedToBase): + DEPTYPE_CHOICES = ( + ('D', 'Depend'), + ('O', 'Optional Depend'), + ('M', 'Make Depend'), + ('C', 'Check Depend'), + ) + pkg = models.ForeignKey(Package, related_name='depends') comparison = models.CharField(max_length=255, default='') optional = models.BooleanField(default=False) description = models.TextField(null=True, blank=True) + deptype = models.CharField(max_length=1, default='D', + choices=DEPTYPE_CHOICES) class Conflict(RelatedToBase): -- cgit v1.2.3-2-g168b From 4c02b11cd6c53e122e1c919b64e28646960c5eda Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 31 Jul 2012 20:29:53 -0500 Subject: Remove optional package depends column This is now completely replaced by the deptype column. Signed-off-by: Dan McGee --- packages/models.py | 1 - 1 file changed, 1 deletion(-) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index b3752b6c..65aa8f4a 100644 --- a/packages/models.py +++ b/packages/models.py @@ -412,7 +412,6 @@ class Depend(RelatedToBase): pkg = models.ForeignKey(Package, related_name='depends') comparison = models.CharField(max_length=255, default='') - optional = models.BooleanField(default=False) description = models.TextField(null=True, blank=True) deptype = models.CharField(max_length=1, default='D', choices=DEPTYPE_CHOICES) -- cgit v1.2.3-2-g168b From f61e61c8a6fc0753359963a836bf65a3a8b1981e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 4 Aug 2012 15:07:03 -0500 Subject: Include description in Depend unicode() output This overrides the base class __unicode__ method. Signed-off-by: Dan McGee --- packages/models.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index 65aa8f4a..1959183f 100644 --- a/packages/models.py +++ b/packages/models.py @@ -416,6 +416,13 @@ class Depend(RelatedToBase): deptype = models.CharField(max_length=1, default='D', choices=DEPTYPE_CHOICES) + def __unicode__(self): + '''For depends, we may also have a description and a modifier.''' + to_str = super(Depend, self).__unicode__() + if self.description: + return u'%s: %s' % (to_str, self.description) + return to_str + class Conflict(RelatedToBase): pkg = models.ForeignKey(Package, related_name='conflicts') -- cgit v1.2.3-2-g168b From f63a70f5118781fe34d82ae0d59c911f0ea28d1d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 7 Aug 2012 19:36:07 -0500 Subject: Make use of new ctypes ALPM API We can use this when filtering down lists of depends, required by, conflicts, etc. to ensure we are honoring the version specifications the same way pacman would. Signed-off-by: Dan McGee --- packages/models.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index 1959183f..403dfef0 100644 --- a/packages/models.py +++ b/packages/models.py @@ -7,6 +7,7 @@ from django.contrib.auth.models import User from main.models import Arch, Repo, Package from main.utils import set_created_field, database_vendor +from packages.alpm import AlpmAPI class PackageRelation(models.Model): @@ -341,6 +342,13 @@ class RelatedToBase(models.Model): # make sure we match architectures if possible arches = self.pkg.applicable_arches() pkgs = pkgs.filter(arch__in=arches) + # if we have a comparison operation, make sure the packages we grab + # actually satisfy the requirements + if self.comparison and self.version: + alpm = AlpmAPI() + pkgs = [pkg for pkg in pkgs if not alpm.available or + alpm.compare_versions(pkg.full_version, self.comparison, + self.version)] if len(pkgs) == 0: # couldn't find a package in the DB # it should be a virtual depend (or a removed package) @@ -373,6 +381,21 @@ class RelatedToBase(models.Model): arches = self.pkg.applicable_arches() pkgs = pkgs.filter(arch__in=arches) + # If we have a comparison operation, make sure the packages we grab + # actually satisfy the requirements. + alpm = AlpmAPI() + if alpm.available and self.comparison and self.version: + pkgs = pkgs.prefetch_related('provides') + new_pkgs = [] + for package in pkgs: + for provide in package.provides.all(): + if provide.name != self.name: + continue + if alpm.compare_versions(provide.version, + self.comparison, self.version): + new_pkgs.append(package) + pkgs = new_pkgs + # Logic here is to filter out packages that are in multiple repos if # they are not requested. For example, if testing is False, only show a # testing package if it doesn't exist in a non-testing repo. -- cgit v1.2.3-2-g168b From b7a03d89d126989bf53005404759482e17163991 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 7 Aug 2012 23:22:18 -0500 Subject: Push more default values down into the database This makes it easier to do manual manipulation/insertion/etc. at the database level, as well as just making things act more sane from an overall software stack perspective. Signed-off-by: Dan McGee --- packages/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index 403dfef0..11fd0a66 100644 --- a/packages/models.py +++ b/packages/models.py @@ -176,7 +176,7 @@ class FlagRequest(models.Model): ip_address = models.GenericIPAddressField(verbose_name='IP address', unpack_ipv4=True) pkgbase = models.CharField(max_length=255, db_index=True) - version = models.CharField(max_length=255, default='') + version = models.CharField(max_length=255) repo = models.ForeignKey(Repo) num_packages = models.PositiveIntegerField('number of packages', default=1) message = models.TextField('message to developer', blank=True) -- cgit v1.2.3-2-g168b From 411ccfb3c74c521969ca6b68459289134976547d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 8 Aug 2012 22:04:07 -0500 Subject: Begin split of flag request version column into parts Not sure why on only this one I decided to put all three parts in the same column. We don't do this anywhere else. Signed-off-by: Dan McGee --- packages/models.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index 11fd0a66..03da27ec 100644 --- a/packages/models.py +++ b/packages/models.py @@ -177,6 +177,9 @@ class FlagRequest(models.Model): unpack_ipv4=True) pkgbase = models.CharField(max_length=255, db_index=True) version = models.CharField(max_length=255) + pkgver = models.CharField(max_length=255) + pkgrel = models.CharField(max_length=255) + epoch = models.PositiveIntegerField(default=0) repo = models.ForeignKey(Repo) num_packages = models.PositiveIntegerField('number of packages', default=1) message = models.TextField('message to developer', blank=True) @@ -193,6 +196,17 @@ class FlagRequest(models.Model): return self.user.get_full_name() return self.user_email + @property + def full_version(self): + # Difference here from other implementations at the moment: we need to + # handle the case of pkgver and pkgrel being null as this table didn't + # originally have version columns. + if self.pkgver == '' and self.pkgrel == '': + return u'' + if self.epoch > 0: + return u'%d:%s-%s' % (self.epoch, self.pkgver, self.pkgrel) + return u'%s-%s' % (self.pkgver, self.pkgrel) + def __unicode__(self): return u'%s from %s on %s' % (self.pkgbase, self.who(), self.created) -- cgit v1.2.3-2-g168b From 32e3e24bbc26a44cbf6fce06cf802ee26f81aa48 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 8 Aug 2012 22:22:24 -0500 Subject: Drop old flag request version column Signed-off-by: Dan McGee --- packages/models.py | 1 - 1 file changed, 1 deletion(-) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index 03da27ec..af428a77 100644 --- a/packages/models.py +++ b/packages/models.py @@ -176,7 +176,6 @@ class FlagRequest(models.Model): ip_address = models.GenericIPAddressField(verbose_name='IP address', unpack_ipv4=True) pkgbase = models.CharField(max_length=255, db_index=True) - version = models.CharField(max_length=255) pkgver = models.CharField(max_length=255) pkgrel = models.CharField(max_length=255) epoch = models.PositiveIntegerField(default=0) -- cgit v1.2.3-2-g168b From 9189665c2c64c13a59b25b981633ae9e12cfcc92 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 15 Aug 2012 08:24:10 -0500 Subject: Ensure created is set when creating flag request via admin Signed-off-by: Dan McGee --- packages/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index af428a77..0bea21b1 100644 --- a/packages/models.py +++ b/packages/models.py @@ -482,7 +482,8 @@ class Replacement(RelatedToBase): # hook up some signals -for sender in (PackageRelation, SignoffSpecification, Signoff, Update): +for sender in (FlagRequest, PackageRelation, + SignoffSpecification, Signoff, Update): pre_save.connect(set_created_field, sender=sender, dispatch_uid="packages.models") -- cgit v1.2.3-2-g168b From bdee24b9d1279de67dd238e3644c2efff314bd7b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 26 Oct 2012 17:11:11 -0500 Subject: Cleanup meta model attributes Signed-off-by: Dan McGee --- packages/models.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index 0bea21b1..0d0fbdf2 100644 --- a/packages/models.py +++ b/packages/models.py @@ -329,6 +329,9 @@ class PackageGroup(models.Model): def __unicode__(self): return "%s: %s" % (self.name, self.pkg) + class Meta: + ordering = ('name',) + class License(models.Model): pkg = models.ForeignKey(Package, related_name='licenses') @@ -338,7 +341,7 @@ class License(models.Model): return self.name class Meta: - ordering = ['name'] + ordering = ('name',) class RelatedToBase(models.Model): @@ -435,7 +438,7 @@ class RelatedToBase(models.Model): class Meta: abstract = True - ordering = ['name'] + ordering = ('name',) class Depend(RelatedToBase): -- cgit v1.2.3-2-g168b From 9e9157d0a8cbf9ea076231e438fb30f58bff8e29 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 16 Nov 2012 16:37:31 -0600 Subject: Use python set comprehension syntax supported in 2.7 Signed-off-by: Dan McGee --- packages/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index 0d0fbdf2..ede8c275 100644 --- a/packages/models.py +++ b/packages/models.py @@ -33,7 +33,7 @@ class PackageRelation(models.Model): def repositories(self): packages = self.get_associated_packages() - return sorted(set([p.repo for p in packages])) + return sorted({p.repo for p in packages}) def __unicode__(self): return u'%s: %s (%s)' % ( -- cgit v1.2.3-2-g168b From e08096ee214b1fd60d093e2902c6acec9cf4ae5f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 11 Dec 2012 21:19:16 -0600 Subject: Fix FS#32018, provides links always go to [testing] packages Remove some of the smarts and do less, but be better about properly sorting the items as one might expect. Signed-off-by: Dan McGee --- packages/models.py | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index ede8c275..a4095f53 100644 --- a/packages/models.py +++ b/packages/models.py @@ -412,24 +412,12 @@ class RelatedToBase(models.Model): new_pkgs.append(package) pkgs = new_pkgs - # Logic here is to filter out packages that are in multiple repos if - # they are not requested. For example, if testing is False, only show a - # testing package if it doesn't exist in a non-testing repo. - filtered = {} - for package in pkgs: - if package.pkgname not in filtered or \ - package.repo.staging == self.pkg.repo.staging: - filtered[package.pkgname] = package - pkgs = filtered.values() - - filtered = {} - for package in pkgs: - if package.pkgname not in filtered or \ - package.repo.testing == self.pkg.repo.testing: - filtered[package.pkgname] = package - pkgs = filtered.values() - - return pkgs + # Sort providers by preference. We sort those in same staging/testing + # combination first, followed by others. We sort by a (staging, + # testing) match tuple that will be (True, True) in the best case. + key_func = lambda x: (x.repo.staging == self.pkg.repo.staging, + x.repo.testing == self.pkg.repo.testing) + return sorted(pkgs, key=key_func, reverse=True) def __unicode__(self): if self.version: -- cgit v1.2.3-2-g168b From 20b64e42672d185821cc584dfa4b133ee259a144 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 28 Dec 2012 14:41:18 -0600 Subject: Reduce query count when retrieving satisfiers and providers Django doesn't attach the parent object to the child objects, even when queried through the related manager. This causes up to 3 extra queries: one to retrieve the package again, and one each for arch and repo retrieval. For a package like archboot, this drops the number of necessary queries for the package page from 805 to 222 (yes, this is still too high) and cuts the time spent waiting on the database from 505ms to 262ms. Signed-off-by: Dan McGee --- packages/models.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index a4095f53..18d57c58 100644 --- a/packages/models.py +++ b/packages/models.py @@ -349,14 +349,16 @@ class RelatedToBase(models.Model): name = models.CharField(max_length=255, db_index=True) version = models.CharField(max_length=255, default='') - def get_best_satisfier(self): + def get_best_satisfier(self, main_pkg=None): '''Find a satisfier for this related package that best matches the given criteria. It will not search provisions, but will find packages named and matching repo characteristics if possible.''' + if main_pkg is None: + main_pkg = self.pkg pkgs = Package.objects.normal().filter(pkgname=self.name) - if not self.pkg.arch.agnostic: + if not main_pkg.arch.agnostic: # make sure we match architectures if possible - arches = self.pkg.applicable_arches() + arches = main_pkg.applicable_arches() pkgs = pkgs.filter(arch__in=arches) # if we have a comparison operation, make sure the packages we grab # actually satisfy the requirements @@ -376,25 +378,27 @@ class RelatedToBase(models.Model): pkg = pkgs[0] # prevents yet more DB queries, these lists should be short; # after each grab the best available in case we remove all entries - pkgs = [p for p in pkgs if p.repo.staging == self.pkg.repo.staging] + pkgs = [p for p in pkgs if p.repo.staging == main_pkg.repo.staging] if len(pkgs) > 0: pkg = pkgs[0] - pkgs = [p for p in pkgs if p.repo.testing == self.pkg.repo.testing] + pkgs = [p for p in pkgs if p.repo.testing == main_pkg.repo.testing] if len(pkgs) > 0: pkg = pkgs[0] return pkg - def get_providers(self): + def get_providers(self, main_pkg=None): '''Return providers of this related package. Does *not* include exact matches as it checks the Provision names only, use get_best_satisfier() instead for exact matches.''' + if main_pkg is None: + main_pkg = self.pkg pkgs = Package.objects.normal().filter( provides__name=self.name).order_by().distinct() - if not self.pkg.arch.agnostic: + if not main_pkg.arch.agnostic: # make sure we match architectures if possible - arches = self.pkg.applicable_arches() + arches = main_pkg.applicable_arches() pkgs = pkgs.filter(arch__in=arches) # If we have a comparison operation, make sure the packages we grab -- cgit v1.2.3-2-g168b From 255a992c151b1c9386c3450083144447be2ce27a Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 9 Jan 2013 08:02:19 -0500 Subject: Add get_associated_packages method to FlagRequest Signed-off-by: Dan McGee --- packages/models.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index 18d57c58..ef86d8e9 100644 --- a/packages/models.py +++ b/packages/models.py @@ -206,6 +206,13 @@ class FlagRequest(models.Model): return u'%d:%s-%s' % (self.epoch, self.pkgver, self.pkgrel) return u'%s-%s' % (self.pkgver, self.pkgrel) + def get_associated_packages(self): + return Package.objects.normal().filter( + pkgbase=self.pkgbase, + repo__testing=self.repo.testing, + repo__staging=self.repo.staging).order_by( + 'pkgname', 'repo__name', 'arch__name') + def __unicode__(self): return u'%s from %s on %s' % (self.pkgbase, self.who(), self.created) -- cgit v1.2.3-2-g168b From be49f26a815cca589c625ff8dd99c85a80262281 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 22 Jan 2013 20:58:50 -0700 Subject: Slight optimization when searching for removed package Signed-off-by: Dan McGee --- packages/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index ef86d8e9..ff677883 100644 --- a/packages/models.py +++ b/packages/models.py @@ -318,7 +318,8 @@ class Update(models.Model): return u'%s-%s' % (self.new_pkgver, self.new_pkgrel) def elsewhere(self): - return Package.objects.filter(pkgname=self.pkgname, arch=self.arch) + return Package.objects.normal().filter( + pkgname=self.pkgname, arch=self.arch) def __unicode__(self): return u'%s of %s on %s' % (self.get_action_flag_display(), -- cgit v1.2.3-2-g168b From 271d1babbf8038e17d9dc5cfc3cd659463848400 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 20 Jan 2013 12:43:33 -0600 Subject: Revert "Reduce query count when retrieving satisfiers and providers" This reverts commit 20b64e42672d185821cc584dfa4b133ee259a144. Django 1.5 fixed this issue and now parent objects are automatically attached to their children when queries go through the related manager. See "Caching of related model instances" in the release notes. Signed-off-by: Dan McGee --- packages/models.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) (limited to 'packages/models.py') diff --git a/packages/models.py b/packages/models.py index ff677883..7bcdc000 100644 --- a/packages/models.py +++ b/packages/models.py @@ -357,16 +357,14 @@ class RelatedToBase(models.Model): name = models.CharField(max_length=255, db_index=True) version = models.CharField(max_length=255, default='') - def get_best_satisfier(self, main_pkg=None): + def get_best_satisfier(self): '''Find a satisfier for this related package that best matches the given criteria. It will not search provisions, but will find packages named and matching repo characteristics if possible.''' - if main_pkg is None: - main_pkg = self.pkg pkgs = Package.objects.normal().filter(pkgname=self.name) - if not main_pkg.arch.agnostic: + if not self.pkg.arch.agnostic: # make sure we match architectures if possible - arches = main_pkg.applicable_arches() + arches = self.pkg.applicable_arches() pkgs = pkgs.filter(arch__in=arches) # if we have a comparison operation, make sure the packages we grab # actually satisfy the requirements @@ -386,27 +384,25 @@ class RelatedToBase(models.Model): pkg = pkgs[0] # prevents yet more DB queries, these lists should be short; # after each grab the best available in case we remove all entries - pkgs = [p for p in pkgs if p.repo.staging == main_pkg.repo.staging] + pkgs = [p for p in pkgs if p.repo.staging == self.pkg.repo.staging] if len(pkgs) > 0: pkg = pkgs[0] - pkgs = [p for p in pkgs if p.repo.testing == main_pkg.repo.testing] + pkgs = [p for p in pkgs if p.repo.testing == self.pkg.repo.testing] if len(pkgs) > 0: pkg = pkgs[0] return pkg - def get_providers(self, main_pkg=None): + def get_providers(self): '''Return providers of this related package. Does *not* include exact matches as it checks the Provision names only, use get_best_satisfier() instead for exact matches.''' - if main_pkg is None: - main_pkg = self.pkg pkgs = Package.objects.normal().filter( provides__name=self.name).order_by().distinct() - if not main_pkg.arch.agnostic: + if not self.pkg.arch.agnostic: # make sure we match architectures if possible - arches = main_pkg.applicable_arches() + arches = self.pkg.applicable_arches() pkgs = pkgs.filter(arch__in=arches) # If we have a comparison operation, make sure the packages we grab -- cgit v1.2.3-2-g168b