From badc535aeb1d310a9b8aa59aade07045e6eae653 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 18 Apr 2012 15:05:43 -0500 Subject: Ensure order_by default value is cleared when using distinct() Otherwise the queryset returns nonsensical results. I find the design of this less than obvious but so be it; we can ensure the results work regardless of a default ordering on the model. Signed-off-by: Dan McGee --- main/models.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index c532ed56..398cb88b 100644 --- a/main/models.py +++ b/main/models.py @@ -13,7 +13,7 @@ from .utils import cache_function, set_created_field, utc_now class TodolistManager(models.Manager): def incomplete(self): - return self.filter(todolistpkg__complete=False).distinct() + return self.filter(todolistpkg__complete=False).order_by().distinct() class PackageManager(models.Manager): def flagged(self): @@ -378,7 +378,7 @@ class PackageDepend(models.Model): '''Return providers of this dep. Does *not* include exact matches as it checks the Provision names only, use get_best_satisfier() instead.''' pkgs = Package.objects.normal().filter( - provides__name=self.depname).distinct() + provides__name=self.depname).order_by().distinct() if arches is not None: pkgs = pkgs.filter(arch__in=arches) @@ -432,7 +432,8 @@ class Todolist(models.Model): @property def package_names(self): # depends on packages property returning a queryset - return self.packages.values_list('pkg__pkgname', flat=True).distinct() + return self.packages.values_list( + 'pkg__pkgname', flat=True).order_by().distinct() class Meta: db_table = 'todolists' -- cgit v1.2.3-2-g168b From f6c2bc6c33bf1fe4fe4cfff4c0177fd296c3b587 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 18 May 2012 19:55:48 -0500 Subject: Simplify get_best_satisfier and get_providers We always passed values in that came off the containing package object; we can access these directly in the methods themselves. Signed-off-by: Dan McGee --- main/models.py | 51 ++++++++++++++++++++++++--------------------------- 1 file changed, 24 insertions(+), 27 deletions(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index 398cb88b..4b445dd0 100644 --- a/main/models.py +++ b/main/models.py @@ -231,16 +231,12 @@ class Package(models.Model): """ deps = [] arches = None - if not self.arch.agnostic: - arches = self.applicable_arches() # TODO: we can use list comprehension and an 'in' query to make this more effective for dep in self.depends.order_by('optional', 'depname'): - pkg = dep.get_best_satisfier(arches, testing=self.repo.testing, - staging=self.repo.staging) + pkg = dep.get_best_satisfier() providers = None if not pkg: - providers = dep.get_providers(arches, - testing=self.repo.testing, staging=self.repo.staging) + providers = dep.get_providers() deps.append({'dep': dep, 'pkg': pkg, 'providers': providers}) return deps @@ -343,13 +339,14 @@ class PackageDepend(models.Model): optional = models.BooleanField(default=False) description = models.TextField(null=True, blank=True) - def get_best_satisfier(self, arches=None, testing=None, staging=None): + def get_best_satisfier(self): '''Find a satisfier for this dependency that best matches the given criteria. It will not search provisions, but will find packages named and matching repo characteristics if possible.''' pkgs = Package.objects.normal().filter(pkgname=self.depname) - if arches is not None: + if not self.pkg.arch.agnostic: # make sure we match architectures if possible + arches = self.pkg.applicable_arches() pkgs = pkgs.filter(arch__in=arches) if len(pkgs) == 0: # couldn't find a package in the DB @@ -362,42 +359,42 @@ class PackageDepend(models.Model): pkg = pkgs[0] # prevents yet more DB queries, these lists should be short; # after each grab the best available in case we remove all entries - if staging is not None: - pkgs = [p for p in pkgs if p.repo.staging == staging] + pkgs = [p for p in pkgs if p.repo.staging == self.pkg.repo.staging] if len(pkgs) > 0: pkg = pkgs[0] - if testing is not None: - pkgs = [p for p in pkgs if p.repo.testing == testing] + pkgs = [p for p in pkgs if p.repo.testing == self.pkg.repo.testing] if len(pkgs) > 0: pkg = pkgs[0] return pkg - def get_providers(self, arches=None, testing=None, staging=None): + def get_providers(self): '''Return providers of this dep. Does *not* include exact matches as it checks the Provision names only, use get_best_satisfier() instead.''' pkgs = Package.objects.normal().filter( provides__name=self.depname).order_by().distinct() - if arches is not None: + if not self.pkg.arch.agnostic: + # make sure we match architectures if possible + arches = self.pkg.applicable_arches() pkgs = pkgs.filter(arch__in=arches) # Logic here is to filter out packages that are in multiple repos if # they are not requested. For example, if testing is False, only show a # testing package if it doesn't exist in a non-testing repo. - if staging is not None: - filtered = {} - for p in pkgs: - if p.pkgname not in filtered or p.repo.staging == staging: - filtered[p.pkgname] = p - pkgs = filtered.values() - - if testing is not None: - filtered = {} - for p in pkgs: - if p.pkgname not in filtered or p.repo.testing == testing: - filtered[p.pkgname] = p - pkgs = filtered.values() + filtered = {} + for package in pkgs: + if package.pkgname not in filtered or \ + package.repo.staging == self.pkg.repo.staging: + filtered[package.pkgname] = package + pkgs = filtered.values() + + filtered = {} + for package in pkgs: + if package.pkgname not in filtered or \ + package.repo.testing == self.pkg.repo.testing: + filtered[package.pkgname] = package + pkgs = filtered.values() return pkgs -- cgit v1.2.3-2-g168b From 72a92102df4999dbcc370064707c9026d51c4fe7 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 18 May 2012 21:29:03 -0500 Subject: Switch to usage of new Depend object Signed-off-by: Dan McGee --- main/models.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index 4b445dd0..f17d4a4d 100644 --- a/main/models.py +++ b/main/models.py @@ -180,11 +180,12 @@ class Package(models.Model): list slim by including the corresponding package in the same testing category as this package if that check makes sense. """ + from packages.models import Depend provides = set(self.provides.values_list('name', flat=True)) provides.add(self.pkgname) - requiredby = PackageDepend.objects.select_related('pkg', + requiredby = Depend.objects.select_related('pkg', 'pkg__arch', 'pkg__repo').filter( - depname__in=provides).order_by( + name__in=provides).order_by( 'pkg__pkgname', 'pkg__arch__name', 'pkg__repo__name') if not self.arch.agnostic: # make sure we match architectures if possible @@ -232,7 +233,7 @@ class Package(models.Model): deps = [] arches = None # TODO: we can use list comprehension and an 'in' query to make this more effective - for dep in self.depends.order_by('optional', 'depname'): + for dep in self.depends.order_by('optional', 'name'): pkg = dep.get_best_satisfier() providers = None if not pkg: -- cgit v1.2.3-2-g168b From e1f9a3c44a1449a36f3981b868814f3d1f65d70d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 18 May 2012 21:35:28 -0500 Subject: Drop old PackageDepend model Signed-off-by: Dan McGee --- main/migrations/0061_auto__del_packagedepend.py | 135 ++++++++++++++++++++++++ main/models.py | 71 ------------- 2 files changed, 135 insertions(+), 71 deletions(-) create mode 100644 main/migrations/0061_auto__del_packagedepend.py (limited to 'main') diff --git a/main/migrations/0061_auto__del_packagedepend.py b/main/migrations/0061_auto__del_packagedepend.py new file mode 100644 index 00000000..6cb1f68f --- /dev/null +++ b/main/migrations/0061_auto__del_packagedepend.py @@ -0,0 +1,135 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + depends_on = ( + ('packages', '0016_copy_depends_data.py'), + ) + + def forwards(self, orm): + db.delete_table('package_depends') + + def backwards(self, orm): + db.create_table('package_depends', ( + ('description', self.gf('django.db.models.fields.TextField')(null=True, blank=True)), + ('depvcmp', self.gf('django.db.models.fields.CharField')(default='', max_length=255)), + ('pkg', self.gf('django.db.models.fields.related.ForeignKey')(related_name='depends', to=orm['main.Package'])), + ('depname', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)), + ('optional', self.gf('django.db.models.fields.BooleanField')(default=False)), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + )) + db.send_create_signal('main', ['PackageDepend']) + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'main.arch': { + 'Meta': {'ordering': "['name']", 'object_name': 'Arch', 'db_table': "'arches'"}, + 'agnostic': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'main.donor': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Donor', 'db_table': "'donors'"}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'visible': ('django.db.models.fields.BooleanField', [], {'default': 'True'}) + }, + 'main.package': { + 'Meta': {'ordering': "('pkgname',)", 'unique_together': "(('pkgname', 'repo', 'arch'),)", 'object_name': 'Package', 'db_table': "'packages'"}, + 'arch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'on_delete': 'models.PROTECT', 'to': "orm['main.Arch']"}), + 'build_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'compressed_size': ('main.fields.PositiveBigIntegerField', [], {}), + 'epoch': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}), + 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'files_last_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'flag_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'installed_size': ('main.fields.PositiveBigIntegerField', [], {}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'packager': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'on_delete': 'models.SET_NULL'}), + 'packager_str': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pgp_signature': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'pkgdesc': ('django.db.models.fields.TextField', [], {'null': 'True'}), + 'pkgname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'repo': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'on_delete': 'models.PROTECT', 'to': "orm['main.Repo']"}), + 'url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}) + }, + 'main.packagefile': { + 'Meta': {'object_name': 'PackageFile', 'db_table': "'package_files'"}, + 'directory': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_directory': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"}) + }, + 'main.repo': { + 'Meta': {'ordering': "['name']", 'object_name': 'Repo', 'db_table': "'repos'"}, + 'bugs_category': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'bugs_project': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'staging': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'svn_root': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'testing': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + }, + 'main.todolist': { + 'Meta': {'object_name': 'Todolist', 'db_table': "'todolists'"}, + 'creator': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'on_delete': 'models.PROTECT'}), + 'date_added': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'description': ('django.db.models.fields.TextField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + }, + 'main.todolistpkg': { + 'Meta': {'unique_together': "(('list', 'pkg'),)", 'object_name': 'TodolistPkg', 'db_table': "'todolist_pkgs'"}, + 'complete': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'list': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Todolist']"}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"}) + } + } + + complete_apps = ['main'] diff --git a/main/models.py b/main/models.py index f17d4a4d..04d8da8f 100644 --- a/main/models.py +++ b/main/models.py @@ -333,77 +333,6 @@ class PackageFile(models.Model): class Meta: db_table = 'package_files' -class PackageDepend(models.Model): - pkg = models.ForeignKey(Package, related_name='depends') - depname = models.CharField(max_length=255, db_index=True) - depvcmp = models.CharField(max_length=255, default='') - optional = models.BooleanField(default=False) - description = models.TextField(null=True, blank=True) - - def get_best_satisfier(self): - '''Find a satisfier for this dependency that best matches the given - criteria. It will not search provisions, but will find packages named - and matching repo characteristics if possible.''' - pkgs = Package.objects.normal().filter(pkgname=self.depname) - if not self.pkg.arch.agnostic: - # make sure we match architectures if possible - arches = self.pkg.applicable_arches() - pkgs = pkgs.filter(arch__in=arches) - if len(pkgs) == 0: - # couldn't find a package in the DB - # it should be a virtual depend (or a removed package) - return None - if len(pkgs) == 1: - return pkgs[0] - # more than one package, see if we can't shrink it down - # grab the first though in case we fail - pkg = pkgs[0] - # prevents yet more DB queries, these lists should be short; - # after each grab the best available in case we remove all entries - pkgs = [p for p in pkgs if p.repo.staging == self.pkg.repo.staging] - if len(pkgs) > 0: - pkg = pkgs[0] - - pkgs = [p for p in pkgs if p.repo.testing == self.pkg.repo.testing] - if len(pkgs) > 0: - pkg = pkgs[0] - - return pkg - - def get_providers(self): - '''Return providers of this dep. Does *not* include exact matches as it - checks the Provision names only, use get_best_satisfier() instead.''' - pkgs = Package.objects.normal().filter( - provides__name=self.depname).order_by().distinct() - if not self.pkg.arch.agnostic: - # make sure we match architectures if possible - arches = self.pkg.applicable_arches() - pkgs = pkgs.filter(arch__in=arches) - - # Logic here is to filter out packages that are in multiple repos if - # they are not requested. For example, if testing is False, only show a - # testing package if it doesn't exist in a non-testing repo. - filtered = {} - for package in pkgs: - if package.pkgname not in filtered or \ - package.repo.staging == self.pkg.repo.staging: - filtered[package.pkgname] = package - pkgs = filtered.values() - - filtered = {} - for package in pkgs: - if package.pkgname not in filtered or \ - package.repo.testing == self.pkg.repo.testing: - filtered[package.pkgname] = package - pkgs = filtered.values() - - return pkgs - - def __unicode__(self): - return "%s%s" % (self.depname, self.depvcmp) - - class Meta: - db_table = 'package_depends' class Todolist(models.Model): creator = models.ForeignKey(User, on_delete=models.PROTECT) -- cgit v1.2.3-2-g168b From d685d51e8c80f29d4f14977a0d8088534e6626c4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 20 May 2012 13:21:59 -0500 Subject: Ensure we use last_modified date from News in headers We were actually using the postdate attribute rather than last_modified, which means any News objects that get edited would not trigger an update of the feed. Signed-off-by: Dan McGee --- main/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'main') diff --git a/main/utils.py b/main/utils.py index e7e47c53..b7cb19f4 100644 --- a/main/utils.py +++ b/main/utils.py @@ -72,7 +72,7 @@ def refresh_latest(**kwargs): cache.set(cache_key, None, INVALIDATE_TIMEOUT) -def retrieve_latest(sender): +def retrieve_latest(sender, latest_by=None): # we could break this down based on the request url, but it would probably # cost us more in query time to do so. cache_key = CACHE_LATEST_PREFIX + sender.__name__ @@ -80,8 +80,9 @@ def retrieve_latest(sender): if latest: return latest try: - latest_by = sender._meta.get_latest_by - latest = sender.objects.values(latest_by).latest()[latest_by] + if latest_by is None: + latest_by = sender._meta.get_latest_by + latest = sender.objects.values(latest_by).latest(latest_by)[latest_by] # Using add means "don't overwrite anything in there". What could be in # there is an explicit None value that our refresh signal set, which # means we want to avoid race condition possibilities for a bit. -- cgit v1.2.3-2-g168b From 26a00cadcebc0b37775954d261ec73f927ceca12 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 7 Jul 2012 17:28:02 -0500 Subject: Don't log package updates in Python when we have DB trigger support This adds a helper method to find the database engine in use, and then skips code we shouldn't execute if we are doing this another way. Note that this helper method could be useful for backend-specific code paths elsewhere, such as custom SQL being called or lack of StdDev() in sqlite3 out of the box. Signed-off-by: Dan McGee --- main/utils.py | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'main') diff --git a/main/utils.py b/main/utils.py index b7cb19f4..879abfb9 100644 --- a/main/utils.py +++ b/main/utils.py @@ -8,6 +8,7 @@ import hashlib from pytz import utc from django.core.cache import cache +from django.db import connections, router CACHE_TIMEOUT = 1800 @@ -106,6 +107,16 @@ def set_created_field(sender, **kwargs): obj.created = utc_now() +def database_vendor(model, mode='read'): + if mode == 'read': + database = router.db_for_read(model) + elif mode == 'write': + database = router.db_for_write(model) + else: + raise Exception('Invalid database mode specified') + return connections[database].vendor + + def groupby_preserve_order(iterable, keyfunc): '''Take an iterable and regroup using keyfunc to determine whether items belong to the same group. The order of the iterable is preserved and -- cgit v1.2.3-2-g168b From 8383a071608329c7683f7a710030ce945bd20b4d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 8 Jul 2012 23:30:48 -0500 Subject: Add a new jquery_tablesorter CDN template tag And use it everywhere we were including the file before. This should make updating the version a heck of a lot easier. Signed-off-by: Dan McGee --- main/templatetags/cdn.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'main') diff --git a/main/templatetags/cdn.py b/main/templatetags/cdn.py index ab5d881a..54299823 100644 --- a/main/templatetags/cdn.py +++ b/main/templatetags/cdn.py @@ -17,4 +17,11 @@ def jquery(): link = staticfiles_storage.url(filename) return '' % link + +@register.simple_tag +def jquery_tablesorter(): + filename = 'jquery.tablesorter.min.js' + link = staticfiles_storage.url(filename) + return '' % link + # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From c0bf9e20660cfae7ea8994472555bba23398b598 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 24 Jul 2012 09:19:48 -0500 Subject: Remove custom utc_now() function, use django.utils.timezone.now() This was around from the time when we handled timezones sanely and Django did not; now that we are on 1.4 we no longer need our own code to handle this. Signed-off-by: Dan McGee --- main/models.py | 5 +++-- main/utils.py | 9 ++------- 2 files changed, 5 insertions(+), 9 deletions(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index 04d8da8f..6c9dfe4d 100644 --- a/main/models.py +++ b/main/models.py @@ -6,9 +6,10 @@ from pgpdump import BinaryData from django.db import models from django.contrib.auth.models import User from django.contrib.sites.models import Site +from django.utils.timezone import now from .fields import PositiveBigIntegerField -from .utils import cache_function, set_created_field, utc_now +from .utils import cache_function, set_created_field class TodolistManager(models.Manager): @@ -385,7 +386,7 @@ class TodolistPkg(models.Model): def set_todolist_fields(sender, **kwargs): todolist = kwargs['instance'] if not todolist.date_added: - todolist.date_added = utc_now() + todolist.date_added = now() # connect signals needed to keep cache in line with reality from main.utils import refresh_latest diff --git a/main/utils.py b/main/utils.py index 879abfb9..0b6849a4 100644 --- a/main/utils.py +++ b/main/utils.py @@ -5,10 +5,10 @@ except ImportError: from datetime import datetime import hashlib -from pytz import utc from django.core.cache import cache from django.db import connections, router +from django.utils.timezone import now CACHE_TIMEOUT = 1800 @@ -94,17 +94,12 @@ def retrieve_latest(sender, latest_by=None): return None -def utc_now(): - '''Returns a timezone-aware UTC date representing now.''' - return datetime.utcnow().replace(tzinfo=utc) - - def set_created_field(sender, **kwargs): '''This will set the 'created' field on any object to the current UTC time if it is unset. For use as a pre_save signal handler.''' obj = kwargs['instance'] if hasattr(obj, 'created') and not obj.created: - obj.created = utc_now() + obj.created = now() def database_vendor(model, mode='read'): -- cgit v1.2.3-2-g168b From 2a221fa72fd8491ca1a0633ba62b6a846267b4fc Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 9 Jul 2012 00:40:17 -0500 Subject: Upgrade to jQuery 1.7.2 and a maintained tablesorter This touches a wide variety of files as well as makes updates to some of our own code to be fully compatible. We also use some of the newer locale/accent sorting features of tablesorter to make tables with developer names sort in a more sane fashion. Signed-off-by: Dan McGee --- main/templatetags/cdn.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'main') diff --git a/main/templatetags/cdn.py b/main/templatetags/cdn.py index 54299823..fb78039c 100644 --- a/main/templatetags/cdn.py +++ b/main/templatetags/cdn.py @@ -7,7 +7,7 @@ register = template.Library() @register.simple_tag def jquery(): - version = '1.4.4' + version = '1.7.2' oncdn = getattr(settings, 'CDN_ENABLED', True) if oncdn: link = 'https://ajax.googleapis.com/ajax/libs/jquery/' \ @@ -20,7 +20,8 @@ def jquery(): @register.simple_tag def jquery_tablesorter(): - filename = 'jquery.tablesorter.min.js' + version = '2.3.11' + filename = 'jquery.tablesorter-%s.min.js' % version link = staticfiles_storage.url(filename) return '' % link -- cgit v1.2.3-2-g168b From 0cc369e985dd6376f0367e4b57e980ce14231796 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 31 Jul 2012 00:09:28 -0500 Subject: Update several bits and pieces for staging packages This will prevent [staging] packages from cluttering normal user's view on the website, but allow us to still import everything from this repository for developer use. Signed-off-by: Dan McGee --- main/models.py | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index 6c9dfe4d..577f11c6 100644 --- a/main/models.py +++ b/main/models.py @@ -4,6 +4,7 @@ from itertools import groupby from pgpdump import BinaryData from django.db import models +from django.db.models import Q from django.contrib.auth.models import User from django.contrib.sites.models import Site from django.utils.timezone import now @@ -24,6 +25,12 @@ class PackageManager(models.Manager): def normal(self): return self.select_related('arch', 'repo') + def restricted(self, user=None): + qs = self.normal() + if user is not None and user.is_authenticated: + return qs + return qs.filter(repo__staging=False) + class Donor(models.Model): name = models.CharField(max_length=255, unique=True) visible = models.BooleanField(default=True, @@ -193,22 +200,26 @@ class Package(models.Model): requiredby = requiredby.filter( pkg__arch__in=self.applicable_arches()) # sort out duplicate packages; this happens if something has a double - # versioned dep such as a kernel module + # versioned depend such as a kernel module requiredby = [list(vals)[0] for _, vals in groupby(requiredby, lambda x: x.pkg.id)] + if len(requiredby) == 0: + return requiredby - # find another package by this name in the opposite testing setup - # TODO: figure out staging exclusions too - if not Package.objects.filter(pkgname=self.pkgname, - arch=self.arch).exclude(id=self.id).exclude( - repo__testing=self.repo.testing).exists(): + # find another package by this name in a different testing or staging + # repo; if we can't, we can short-circuit some checks + repo_q = (Q(repo__testing=(not self.repo.testing)) | + Q(repo__staging=(not self.repo.staging))) + if not Package.objects.filter( + repo_q, pkgname=self.pkgname, arch=self.arch + ).exclude(id=self.id).exists(): # there isn't one? short circuit, all required by entries are fine return requiredby trimmed = [] # for each unique package name, try to screen our package list down to - # those packages in the same testing category (yes or no) iff there is - # a package in the same testing category. + # those packages in the same testing and staging category (yes or no) + # iff there is a package in the same testing and staging category. for _, dep_pkgs in groupby(requiredby, lambda x: x.pkg.pkgname): dep_pkgs = list(dep_pkgs) dep = dep_pkgs[0] @@ -254,7 +265,7 @@ class Package(models.Model): return Package.objects.normal().get(arch=self.arch, repo=self.repo, pkgname=self.pkgbase) except Package.DoesNotExist: - # this package might be split across repos? just find one + # this package might be split across repos? find one # that matches the correct [testing] repo flag pkglist = Package.objects.normal().filter(arch=self.arch, repo__testing=self.repo.testing, @@ -271,8 +282,10 @@ class Package(models.Model): repo.testing and repo.staging flags. For any non-split packages, the return value will be an empty list. """ - return Package.objects.normal().filter(arch__in=self.applicable_arches(), - repo__testing=self.repo.testing, repo__staging=self.repo.staging, + return Package.objects.normal().filter( + arch__in=self.applicable_arches(), + repo__testing=self.repo.testing, + repo__staging=self.repo.staging, pkgbase=self.pkgbase).exclude(id=self.id) def flag_request(self): -- cgit v1.2.3-2-g168b From 3f0c024754047d92e8ce4aa4ecf93a06865f8448 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 31 Jul 2012 18:37:30 -0500 Subject: PGP key handling updates * Import signatures for all known keys, not just active developers * Ensure we are only showing and accounting for active developers on the master keys page * Add a new table showing signatures between developers Signed-off-by: Dan McGee --- main/templatetags/pgp.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'main') diff --git a/main/templatetags/pgp.py b/main/templatetags/pgp.py index 50b1aa17..5c9fe511 100644 --- a/main/templatetags/pgp.py +++ b/main/templatetags/pgp.py @@ -39,6 +39,14 @@ def pgp_key_link(key_id, link_text=None): values = (url, format_key(key_id), link_text) return '%s' % values +@register.simple_tag +def user_pgp_key_link(users, key_id): + matched = [user for user in users if user.userprofile.pgp_key and + user.userprofile.pgp_key[-16:] == key_id[-16:]] + if matched and len(matched) == 1: + return pgp_key_link(key_id, matched[0].get_full_name()) + return pgp_key_link(key_id) + @register.filter def pgp_fingerprint(key_id, autoescape=True): if not key_id: -- cgit v1.2.3-2-g168b From a64bbbd4139d91cbbca10d804067cbd87a95872d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 31 Jul 2012 20:27:43 -0500 Subject: Make adjustments for optional -> deptype conversion Very little dealt directly with this field. Signed-off-by: Dan McGee --- main/models.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index 577f11c6..f4ced350 100644 --- a/main/models.py +++ b/main/models.py @@ -245,13 +245,18 @@ class Package(models.Model): deps = [] arches = None # TODO: we can use list comprehension and an 'in' query to make this more effective - for dep in self.depends.order_by('optional', 'name'): + for dep in self.depends.all(): pkg = dep.get_best_satisfier() providers = None if not pkg: providers = dep.get_providers() deps.append({'dep': dep, 'pkg': pkg, 'providers': providers}) - return deps + # sort the list; deptype sorting makes this tricker than expected + sort_order = {'D': 0, 'O': 1, 'M': 2, 'C': 3} + def sort_key(val): + dep = val['dep'] + return (sort_order.get(dep.deptype, 1000), dep.name) + return sorted(deps, key=sort_key) @cache_function(125) def base_package(self): -- cgit v1.2.3-2-g168b From 8ca64af397718f7dda0080467d202c6a70a33c8c Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 5 Aug 2012 11:21:59 -0500 Subject: Smarter handling of multilib packages in "Versions Elsewhere" We can do some manipulation of the pkgname to ensure multilib packages show up here, as well as showing the non-multilib versions in the list when viewing the multilib packages. Signed-off-by: Dan McGee --- main/models.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index f4ced350..7d8ea755 100644 --- a/main/models.py +++ b/main/models.py @@ -336,8 +336,16 @@ class Package(models.Model): def elsewhere(self): '''attempt to locate this package anywhere else, regardless of architecture or repository. Excludes this package from the list.''' + names = [self.pkgname] + if self.pkgname.startswith('lib32-'): + names.append(self.pkgname[6:]) + elif self.pkgname.endswith('-multilib'): + names.append(self.pkgname[:-9]) + else: + names.append('lib32-' + self.pkgname) + names.append(self.pkgname + '-multilib') return Package.objects.normal().filter( - pkgname=self.pkgname).exclude(id=self.id).order_by( + pkgname__in=names).exclude(id=self.id).order_by( 'arch__name', 'repo__name') class PackageFile(models.Model): -- cgit v1.2.3-2-g168b From 7d9ed0b881bd05878e7a54f785c2551bc6e336d6 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 7 Aug 2012 01:16:51 -0500 Subject: Add reverse conflicts to package details This is a place where calling vercmp could come in really handy. Signed-off-by: Dan McGee --- main/models.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'main') diff --git a/main/models.py b/main/models.py index 7d8ea755..b49edde3 100644 --- a/main/models.py +++ b/main/models.py @@ -258,6 +258,16 @@ class Package(models.Model): return (sort_order.get(dep.deptype, 1000), dep.name) return sorted(deps, key=sort_key) + @cache_function(123) + def reverse_conflicts(self): + """ + Returns a list of packages with conflicts against this package. + """ + # TODO: fix this; right now we cheat since we can't do proper version + # number checking without using alpm or vercmp directly. + return Package.objects.filter(conflicts__name=self.pkgname, + conflicts__comparison='').distinct() + @cache_function(125) def base_package(self): """ -- cgit v1.2.3-2-g168b From f63a70f5118781fe34d82ae0d59c911f0ea28d1d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 7 Aug 2012 19:36:07 -0500 Subject: Make use of new ctypes ALPM API We can use this when filtering down lists of depends, required by, conflicts, etc. to ensure we are honoring the version specifications the same way pacman would. Signed-off-by: Dan McGee --- main/models.py | 55 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 7 deletions(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index b49edde3..bb2320e8 100644 --- a/main/models.py +++ b/main/models.py @@ -11,6 +11,7 @@ from django.utils.timezone import now from .fields import PositiveBigIntegerField from .utils import cache_function, set_created_field +from packages.alpm import AlpmAPI class TodolistManager(models.Manager): @@ -189,16 +190,42 @@ class Package(models.Model): category as this package if that check makes sense. """ from packages.models import Depend - provides = set(self.provides.values_list('name', flat=True)) - provides.add(self.pkgname) + provides = self.provides.all() + provide_names = set(provide.name for provide in provides) + provide_names.add(self.pkgname) requiredby = Depend.objects.select_related('pkg', 'pkg__arch', 'pkg__repo').filter( - name__in=provides).order_by( + name__in=provide_names).order_by( 'pkg__pkgname', 'pkg__arch__name', 'pkg__repo__name') if not self.arch.agnostic: # make sure we match architectures if possible requiredby = requiredby.filter( pkg__arch__in=self.applicable_arches()) + + # if we can use ALPM, ensure our returned Depend objects abide by the + # version comparison operators they may specify + alpm = AlpmAPI() + if alpm.available: + new_rqd = [] + for dep in requiredby: + if not dep.comparison or not dep.version: + # no comparisson/version, so always let it through + new_rqd.append(dep) + elif self.pkgname == dep.name: + # depends on this package, so check it directly + if alpm.compare_versions(self.full_version, + dep.comparison, dep.version): + new_rqd.append(dep) + else: + # it must be a provision of ours at this point + for provide in (p for p in provides if p.name == dep.name): + print(provide.version, dep.comparison, dep.version) + if alpm.compare_versions(provide.version, + dep.comparison, dep.version): + new_rqd.append(dep) + break + requiredby = new_rqd + # sort out duplicate packages; this happens if something has a double # versioned depend such as a kernel module requiredby = [list(vals)[0] for _, vals in @@ -263,10 +290,24 @@ class Package(models.Model): """ Returns a list of packages with conflicts against this package. """ - # TODO: fix this; right now we cheat since we can't do proper version - # number checking without using alpm or vercmp directly. - return Package.objects.filter(conflicts__name=self.pkgname, - conflicts__comparison='').distinct() + pkgs = Package.objects.filter(conflicts__name=self.pkgname) + alpm = AlpmAPI() + if not alpm.available: + return pkgs + + # If we can use ALPM, we can filter out items that don't actually + # conflict due to the version specification. + pkgs = pkgs.prefetch_related('conflicts') + new_pkgs = [] + for package in pkgs: + for conflict in package.conflicts.all(): + if conflict.name != self.pkgname: + continue + if not conflict.comparison or not conflict.version \ + or alpm.compare_versions(self.full_version, + conflict.comparison, conflict.version): + new_pkgs.append(package) + return new_pkgs @cache_function(125) def base_package(self): -- cgit v1.2.3-2-g168b From b7a03d89d126989bf53005404759482e17163991 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 7 Aug 2012 23:22:18 -0500 Subject: Push more default values down into the database This makes it easier to do manual manipulation/insertion/etc. at the database level, as well as just making things act more sane from an overall software stack perspective. Signed-off-by: Dan McGee --- main/migrations/0032_auto__add_field_arch_agnostic.py | 2 +- main/migrations/0037_auto__add_field_userprofile_time_zone.py | 2 +- main/migrations/0043_auto__add_field_package_epoch.py | 2 +- main/migrations/0046_auto__add_field_repo_staging.py | 2 +- main/migrations/0048_auto__add_field_repo_bugs_category.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) (limited to 'main') diff --git a/main/migrations/0032_auto__add_field_arch_agnostic.py b/main/migrations/0032_auto__add_field_arch_agnostic.py index ab9b9159..9ccf059d 100644 --- a/main/migrations/0032_auto__add_field_arch_agnostic.py +++ b/main/migrations/0032_auto__add_field_arch_agnostic.py @@ -8,7 +8,7 @@ class Migration(SchemaMigration): def forwards(self, orm): # Adding field 'Arch.agnostic' - db.add_column('arches', 'agnostic', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=False) + db.add_column('arches', 'agnostic', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=True) def backwards(self, orm): # Deleting field 'Arch.agnostic' diff --git a/main/migrations/0037_auto__add_field_userprofile_time_zone.py b/main/migrations/0037_auto__add_field_userprofile_time_zone.py index 9b9b8beb..3a65eacc 100644 --- a/main/migrations/0037_auto__add_field_userprofile_time_zone.py +++ b/main/migrations/0037_auto__add_field_userprofile_time_zone.py @@ -8,7 +8,7 @@ class Migration(SchemaMigration): def forwards(self, orm): # Adding field 'UserProfile.time_zone' - db.add_column('user_profiles', 'time_zone', self.gf('django.db.models.fields.CharField')(default='UTC', max_length=100), keep_default=False) + db.add_column('user_profiles', 'time_zone', self.gf('django.db.models.fields.CharField')(default='UTC', max_length=100), keep_default=True) def backwards(self, orm): # Deleting field 'UserProfile.time_zone' diff --git a/main/migrations/0043_auto__add_field_package_epoch.py b/main/migrations/0043_auto__add_field_package_epoch.py index 77cd9b49..1c6ae9db 100644 --- a/main/migrations/0043_auto__add_field_package_epoch.py +++ b/main/migrations/0043_auto__add_field_package_epoch.py @@ -9,7 +9,7 @@ class Migration(SchemaMigration): def forwards(self, orm): # Adding field 'Package.epoch' - db.add_column('packages', 'epoch', self.gf('django.db.models.fields.PositiveIntegerField')(default=0), keep_default=False) + db.add_column('packages', 'epoch', self.gf('django.db.models.fields.PositiveIntegerField')(default=0), keep_default=True) def backwards(self, orm): diff --git a/main/migrations/0046_auto__add_field_repo_staging.py b/main/migrations/0046_auto__add_field_repo_staging.py index 40c3cb20..0daaf69b 100644 --- a/main/migrations/0046_auto__add_field_repo_staging.py +++ b/main/migrations/0046_auto__add_field_repo_staging.py @@ -7,7 +7,7 @@ from django.db import models class Migration(SchemaMigration): def forwards(self, orm): - db.add_column('repos', 'staging', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=False) + db.add_column('repos', 'staging', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=True) def backwards(self, orm): db.delete_column('repos', 'staging') diff --git a/main/migrations/0048_auto__add_field_repo_bugs_category.py b/main/migrations/0048_auto__add_field_repo_bugs_category.py index 30575126..3e61f7ed 100644 --- a/main/migrations/0048_auto__add_field_repo_bugs_category.py +++ b/main/migrations/0048_auto__add_field_repo_bugs_category.py @@ -7,7 +7,7 @@ from django.db import models class Migration(SchemaMigration): def forwards(self, orm): - db.add_column('repos', 'bugs_category', self.gf('django.db.models.fields.SmallIntegerField')(default=0), keep_default=False) + db.add_column('repos', 'bugs_category', self.gf('django.db.models.fields.SmallIntegerField')(default=2), keep_default=False) def backwards(self, orm): db.delete_column('repos', 'bugs_category') -- cgit v1.2.3-2-g168b From b425b192e12afd0584bbffc9ff1d997a330bcd5a Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 8 Aug 2012 22:21:05 -0500 Subject: Migrate flag request version info to new format Signed-off-by: Dan McGee --- main/models.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index bb2320e8..95e3c42b 100644 --- a/main/models.py +++ b/main/models.py @@ -345,12 +345,16 @@ class Package(models.Model): pkgbase=self.pkgbase).exclude(id=self.id) def flag_request(self): - if not self.flag_date: + if self.flag_date is None: return None from packages.models import FlagRequest try: + # Note that we don't match on pkgrel here; this is because a pkgrel + # bump does not unflag a package so we can still show the same flag + # request from a different pkgrel. request = FlagRequest.objects.filter(pkgbase=self.pkgbase, - repo=self.repo).latest() + repo=self.repo, pkgver=self.pkgver, + epoch=self.epoch).latest() return request except FlagRequest.DoesNotExist: return None -- cgit v1.2.3-2-g168b From 5549b119ea84ffd60f2987610ae35fb393d9625e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 9 Aug 2012 20:31:16 -0500 Subject: Add a rate limiting filter for log messages This should help cut down on the massive amount of emails I receive when things go wrong on the production website. Signed-off-by: Dan McGee --- main/log.py | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 main/log.py (limited to 'main') diff --git a/main/log.py b/main/log.py new file mode 100644 index 00000000..63634874 --- /dev/null +++ b/main/log.py @@ -0,0 +1,71 @@ +# Derived from Django snippets: http://djangosnippets.org/snippets/2242/ +from collections import OrderedDict +from datetime import datetime, timedelta +from hashlib import md5 +import traceback +from pytz import utc + + +class LimitedSizeDict(OrderedDict): + def __init__(self, *args, **kwargs): + self.size_limit = kwargs.pop('size', None) + if self.size_limit == 0: + self.size_limit = None + if self.size_limit and self.size_limit < 0: + raise Exception('Invalid size specified') + super(LimitedSizeDict, self).__init__(*args, **kwargs) + self.check_item_limits() + + def __setitem__(self, key, value): + # delete and add to ensure it ends up at the end of the linked list + if key in self: + super(LimitedSizeDict, self).__delitem__(key) + super(LimitedSizeDict, self).__setitem__(key, value) + self.check_item_limits() + + def check_item_limits(self): + if self.size_limit is None: + return + while len(self) > self.size_limit: + self.popitem(last=False) + + +class RateLimitFilter(object): + def __init__(self, name='', rate=10, prefix='error_rate', max_keys=100): + # delayed import otherwise we have a circular dep when setting up + # the logging config: settings -> logging -> cache -> settings + self.cache_module = __import__('django.core.cache', fromlist=['cache']) + self.errors = LimitedSizeDict(size=max_keys) + self.rate = rate + self.prefix = prefix + + def filter(self, record): + if self.rate == 0: + # rate == 0 means totally unfiltered + return True + + trace = '\n'.join(traceback.format_exception(*record.exc_info)) + key = md5(trace).hexdigest() + duplicate = False + cache = self.cache_module.cache + + # Test if the cache works + try: + cache.set(self.prefix, 1, 300) + use_cache = (cache.get(self.prefix) == 1) + except: + use_cache = False + + if use_cache: + cache_key = '%s_%s' % (self.prefix, key) + duplicate = (cache.get(cache_key) == 1) + cache.set(cache_key, 1, self.rate) + else: + now = datetime.utcnow().replace(tzinfo=utc) + min_date = now - timedelta(seconds=self.rate) + duplicate = (key in self.errors and self.errors[key] >= min_date) + self.errors[key] = now + + return not duplicate + +# vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From 04066d190efddaf890d8573a3fbf4fd1706cb51f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 15 Aug 2012 20:26:23 -0500 Subject: Ensure reverse conflicts match architecture if applicable Signed-off-by: Dan McGee --- main/models.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'main') diff --git a/main/models.py b/main/models.py index 95e3c42b..202e7fa6 100644 --- a/main/models.py +++ b/main/models.py @@ -291,6 +291,10 @@ class Package(models.Model): Returns a list of packages with conflicts against this package. """ pkgs = Package.objects.filter(conflicts__name=self.pkgname) + if not self.arch.agnostic: + # make sure we match architectures if possible + pkgs = pkgs.filter(arch__in=self.applicable_arches()) + alpm = AlpmAPI() if not alpm.available: return pkgs -- cgit v1.2.3-2-g168b From 0ec1af27aeb2ff94128c84ad53f09045cd9ee6e3 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 20 Aug 2012 21:18:18 -0500 Subject: Mark nullable fields as blank on package model This helps when creating test packages through the Django admin. Signed-off-by: Dan McGee --- main/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index 202e7fa6..014464ff 100644 --- a/main/models.py +++ b/main/models.py @@ -105,10 +105,10 @@ class Package(models.Model): last_update = models.DateTimeField(db_index=True) files_last_update = models.DateTimeField(null=True, blank=True) packager_str = models.CharField(max_length=255) - packager = models.ForeignKey(User, null=True, + packager = models.ForeignKey(User, null=True, blank=True, on_delete=models.SET_NULL) pgp_signature = models.TextField(null=True, blank=True) - flag_date = models.DateTimeField(null=True) + flag_date = models.DateTimeField(null=True, blank=True) objects = PackageManager() -- cgit v1.2.3-2-g168b From 78553abc51ba0f18e1c3bec015c9a11d4760c522 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 20 Aug 2012 23:48:44 -0500 Subject: Don't blow up when pgp signature data is '' on a package We handled None/NULL correctly, but not the empty string. Fix this corner case. Signed-off-by: Dan McGee --- main/models.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'main') diff --git a/main/models.py b/main/models.py index 014464ff..42f8e89a 100644 --- a/main/models.py +++ b/main/models.py @@ -143,6 +143,8 @@ class Package(models.Model): data = b64decode(self.pgp_signature) except TypeError: return None + if not data: + return None data = BinaryData(data) packets = list(data.packets()) return packets[0] -- cgit v1.2.3-2-g168b From 0926cf30f4bacf7922c2f27e4f27f78f8182aee0 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 4 Sep 2012 08:22:38 -0500 Subject: Filter out spam flag requests on package details page No need to show these as a matching request. Signed-off-by: Dan McGee --- main/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index 42f8e89a..9ab4b4ad 100644 --- a/main/models.py +++ b/main/models.py @@ -360,7 +360,7 @@ class Package(models.Model): # request from a different pkgrel. request = FlagRequest.objects.filter(pkgbase=self.pkgbase, repo=self.repo, pkgver=self.pkgver, - epoch=self.epoch).latest() + epoch=self.epoch, is_spam=False).latest() return request except FlagRequest.DoesNotExist: return None -- cgit v1.2.3-2-g168b From 5df83a38282d8ddbc653859914eb49cad1c30494 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 4 Sep 2012 08:53:39 -0500 Subject: Add a 'format_http_headers' method This takes a HttpRequest object and grabs the HTTP headers out of it and pretty-prints them in a familiar format. This will come in handy if we want to log these when creating package FlagRequests, releng Tests, etc. in addition to already logging the IP address of the user posting the request. Signed-off-by: Dan McGee --- main/utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'main') diff --git a/main/utils.py b/main/utils.py index 0b6849a4..d12e5e1a 100644 --- a/main/utils.py +++ b/main/utils.py @@ -53,6 +53,16 @@ def clear_cache_function(func, args, kwargs): key = cache_function_key(func, args, kwargs) cache.delete(key) + +def format_http_headers(request): + headers = sorted((k, v) for k, v in request.META.items() + if k.startswith('HTTP_')) + data = [] + for k, v in headers: + data.extend([k[5:].replace('_', '-').title(), ': ', v, '\n']) + return ''.join(data) + + # utility to make a pair of django choices make_choice = lambda l: [(str(m), str(m)) for m in l] -- cgit v1.2.3-2-g168b From 277d71709ce18bfea64e05a6c190a5b6d555d76f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 5 Sep 2012 09:32:24 -0500 Subject: Upgrade jQuery to 1.8.1 Signed-off-by: Dan McGee --- main/templatetags/cdn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'main') diff --git a/main/templatetags/cdn.py b/main/templatetags/cdn.py index fb78039c..a3ed465c 100644 --- a/main/templatetags/cdn.py +++ b/main/templatetags/cdn.py @@ -7,7 +7,7 @@ register = template.Library() @register.simple_tag def jquery(): - version = '1.7.2' + version = '1.8.1' oncdn = getattr(settings, 'CDN_ENABLED', True) if oncdn: link = 'https://ajax.googleapis.com/ajax/libs/jquery/' \ -- cgit v1.2.3-2-g168b From 8bf9147f47341d6efcad20931255888f7186bb7b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 24 Sep 2012 20:24:23 -0500 Subject: Update jQuery to 1.8.2 Signed-off-by: Dan McGee --- main/templatetags/cdn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'main') diff --git a/main/templatetags/cdn.py b/main/templatetags/cdn.py index a3ed465c..b1c65818 100644 --- a/main/templatetags/cdn.py +++ b/main/templatetags/cdn.py @@ -7,7 +7,7 @@ register = template.Library() @register.simple_tag def jquery(): - version = '1.8.1' + version = '1.8.2' oncdn = getattr(settings, 'CDN_ENABLED', True) if oncdn: link = 'https://ajax.googleapis.com/ajax/libs/jquery/' \ -- cgit v1.2.3-2-g168b From f1d194087b717fdf304b1cdd1b54826db83c0c57 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 25 Sep 2012 18:23:07 -0500 Subject: Add staging repos to repos JSON fixture Signed-off-by: Dan McGee --- main/fixtures/repos.json | 60 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) (limited to 'main') diff --git a/main/fixtures/repos.json b/main/fixtures/repos.json index 5fd918bb..a6f564b7 100644 --- a/main/fixtures/repos.json +++ b/main/fixtures/repos.json @@ -11,6 +11,18 @@ "testing": false } }, + { + "pk": 11, + "model": "main.repo", + "fields": { + "bugs_category": 41, + "staging": true, + "name": "Community-Staging", + "bugs_project": 5, + "svn_root": "community", + "testing": false + } + }, { "pk": 6, "model": "main.repo", @@ -47,6 +59,30 @@ "testing": false } }, + { + "pk": 13, + "model": "main.repo", + "fields": { + "bugs_category": 10, + "staging": true, + "name": "Gnome-Unstable", + "bugs_project": 1, + "svn_root": "packages", + "testing": false + } + }, + { + "pk": 12, + "model": "main.repo", + "fields": { + "bugs_category": 10, + "staging": true, + "name": "KDE-Unstable", + "bugs_project": 1, + "svn_root": "packages", + "testing": false + } + }, { "pk": 7, "model": "main.repo", @@ -59,6 +95,18 @@ "testing": false } }, + { + "pk": 14, + "model": "main.repo", + "fields": { + "bugs_category": 46, + "staging": true, + "name": "Multilib-Staging", + "bugs_project": 5, + "svn_root": "community", + "testing": false + } + }, { "pk": 8, "model": "main.repo", @@ -71,6 +119,18 @@ "testing": true } }, + { + "pk": 10, + "model": "main.repo", + "fields": { + "bugs_category": 10, + "staging": true, + "name": "Staging", + "bugs_project": 1, + "svn_root": "packages", + "testing": false + } + }, { "pk": 3, "model": "main.repo", -- cgit v1.2.3-2-g168b From d127ebf0c9303886e326577e0a91565b654e133d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 30 Sep 2012 01:41:43 -0500 Subject: PEP8 cleanups for main/models Signed-off-by: Dan McGee --- main/models.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'main') diff --git a/main/models.py b/main/models.py index 9ab4b4ad..166fd9a9 100644 --- a/main/models.py +++ b/main/models.py @@ -18,6 +18,7 @@ class TodolistManager(models.Manager): def incomplete(self): return self.filter(todolistpkg__complete=False).order_by().distinct() + class PackageManager(models.Manager): def flagged(self): """Used by dev dashboard.""" @@ -32,6 +33,7 @@ class PackageManager(models.Manager): return qs return qs.filter(repo__staging=False) + class Donor(models.Model): name = models.CharField(max_length=255, unique=True) visible = models.BooleanField(default=True, @@ -46,6 +48,7 @@ class Donor(models.Model): ordering = ('name',) get_latest_by = 'when' + class Arch(models.Model): name = models.CharField(max_length=255, unique=True) agnostic = models.BooleanField(default=False, @@ -62,6 +65,7 @@ class Arch(models.Model): ordering = ['name'] verbose_name_plural = 'arches' + class Repo(models.Model): name = models.CharField(max_length=255, unique=True) testing = models.BooleanField(default=False, @@ -86,6 +90,7 @@ class Repo(models.Model): ordering = ['name'] verbose_name_plural = 'repos' + class Package(models.Model): repo = models.ForeignKey(Repo, related_name="packages", on_delete=models.PROTECT) @@ -409,6 +414,7 @@ class Package(models.Model): pkgname__in=names).exclude(id=self.id).order_by( 'arch__name', 'repo__name') + class PackageFile(models.Model): pkg = models.ForeignKey(Package) is_directory = models.BooleanField(default=False) @@ -461,6 +467,7 @@ class Todolist(models.Model): domain = Site.objects.get_current().domain return '%s://%s%s' % (proto, domain, self.get_absolute_url()) + class TodolistPkg(models.Model): list = models.ForeignKey(Todolist) pkg = models.ForeignKey(Package) @@ -470,11 +477,13 @@ class TodolistPkg(models.Model): db_table = 'todolist_pkgs' unique_together = (('list','pkg'),) + def set_todolist_fields(sender, **kwargs): todolist = kwargs['instance'] if not todolist.date_added: todolist.date_added = now() + # connect signals needed to keep cache in line with reality from main.utils import refresh_latest from django.db.models.signals import pre_save, post_save -- cgit v1.2.3-2-g168b From a9998fc09e208b625edc7f39e1d6a49acb533f4b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 30 Sep 2012 09:42:48 -0500 Subject: Remote errant print() statement Signed-off-by: Dan McGee --- main/models.py | 1 - 1 file changed, 1 deletion(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index 166fd9a9..00549268 100644 --- a/main/models.py +++ b/main/models.py @@ -226,7 +226,6 @@ class Package(models.Model): else: # it must be a provision of ours at this point for provide in (p for p in provides if p.name == dep.name): - print(provide.version, dep.comparison, dep.version) if alpm.compare_versions(provide.version, dep.comparison, dep.version): new_rqd.append(dep) -- cgit v1.2.3-2-g168b From 3422efd3073df0b32833aee26a6aee7018010f4e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 18 Oct 2012 08:58:17 -0500 Subject: Modernize initial main migration Signed-off-by: Dan McGee --- main/migrations/0001_initial.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'main') diff --git a/main/migrations/0001_initial.py b/main/migrations/0001_initial.py index 4c89d8a0..bc8bb492 100644 --- a/main/migrations/0001_initial.py +++ b/main/migrations/0001_initial.py @@ -1,9 +1,9 @@ - +# encoding: utf-8 from south.db import db +from south.v2 import SchemaMigration from django.db import models -from main.models import * -class Migration: +class Migration(SchemaMigration): def forwards(self, orm): -- cgit v1.2.3-2-g168b From abd4b6c6d8541955f327b4adc50cf2031ebe6781 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 18 Oct 2012 08:58:44 -0500 Subject: Clean up create index migration Signed-off-by: Dan McGee --- main/migrations/0004_add_pkgname_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'main') diff --git a/main/migrations/0004_add_pkgname_index.py b/main/migrations/0004_add_pkgname_index.py index 0aae4522..6e23adac 100644 --- a/main/migrations/0004_add_pkgname_index.py +++ b/main/migrations/0004_add_pkgname_index.py @@ -7,12 +7,12 @@ class Migration(SchemaMigration): def forwards(self, orm): db.alter_column('packages', 'maintainer_id', orm['main.package:maintainer']) - db.alter_column('packages', 'pkgname', orm['main.package:pkgname']) + db.create_index('packages', ['pkgname']) def backwards(self, orm): db.alter_column('packages', 'maintainer_id', orm['main.package:maintainer']) - db.alter_column('packages', 'pkgname', orm['main.package:pkgname']) + db.delete_index('packages', ['pkgname']) models = { -- cgit v1.2.3-2-g168b From ca1d4faa531768515f48b6856fda13016e12c0d3 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 21 Oct 2012 08:43:27 -0500 Subject: Update tablesorter JS Signed-off-by: Dan McGee --- main/templatetags/cdn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'main') diff --git a/main/templatetags/cdn.py b/main/templatetags/cdn.py index b1c65818..f3ef9a1b 100644 --- a/main/templatetags/cdn.py +++ b/main/templatetags/cdn.py @@ -20,7 +20,7 @@ def jquery(): @register.simple_tag def jquery_tablesorter(): - version = '2.3.11' + version = '2.4.5' filename = 'jquery.tablesorter-%s.min.js' % version link = staticfiles_storage.url(filename) return '' % link -- cgit v1.2.3-2-g168b From bdee24b9d1279de67dd238e3644c2efff314bd7b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 26 Oct 2012 17:11:11 -0500 Subject: Cleanup meta model attributes Signed-off-by: Dan McGee --- main/models.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index 00549268..5700cdf1 100644 --- a/main/models.py +++ b/main/models.py @@ -62,7 +62,7 @@ class Arch(models.Model): class Meta: db_table = 'arches' - ordering = ['name'] + ordering = ('name',) verbose_name_plural = 'arches' @@ -87,8 +87,7 @@ class Repo(models.Model): class Meta: db_table = 'repos' - ordering = ['name'] - verbose_name_plural = 'repos' + ordering = ('name',) class Package(models.Model): -- cgit v1.2.3-2-g168b From 9e9157d0a8cbf9ea076231e438fb30f58bff8e29 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 16 Nov 2012 16:37:31 -0600 Subject: Use python set comprehension syntax supported in 2.7 Signed-off-by: Dan McGee --- main/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index 5700cdf1..cc81637c 100644 --- a/main/models.py +++ b/main/models.py @@ -197,7 +197,7 @@ class Package(models.Model): """ from packages.models import Depend provides = self.provides.all() - provide_names = set(provide.name for provide in provides) + provide_names = {provide.name for provide in provides} provide_names.add(self.pkgname) requiredby = Depend.objects.select_related('pkg', 'pkg__arch', 'pkg__repo').filter( -- cgit v1.2.3-2-g168b From eea25558c766d5f3a32879d16e579d051906cbf3 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 27 Nov 2012 08:48:01 -0600 Subject: Don't cache package properties as aggressively For package signatures, it turns out it is way cheaper to just parse the signature again rather than going though all the decorator and cache_function_key business. This speeds up the mismatched signatures report significantly once this is removed. For base_package, given that we only call it once from our package details template, it makes little sense to cache the result. Signed-off-by: Dan McGee --- main/models.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index cc81637c..603d7ccc 100644 --- a/main/models.py +++ b/main/models.py @@ -141,7 +141,6 @@ class Package(models.Model): return '%s://%s%s' % (proto, domain, self.get_absolute_url()) @property - @cache_function(15) def signature(self): try: data = b64decode(self.pgp_signature) @@ -154,7 +153,6 @@ class Package(models.Model): return packets[0] @property - @cache_function(15) def signer(self): sig = self.signature if sig and sig.key_id: @@ -318,7 +316,6 @@ class Package(models.Model): new_pkgs.append(package) return new_pkgs - @cache_function(125) def base_package(self): """ Locate the base package for this package. It may be this very package, -- cgit v1.2.3-2-g168b From 4c699119820dfd060de6a0385e549f3397053548 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 4 Dec 2012 21:59:29 -0600 Subject: get_latest_by cleanups Fix some that referenced non-existent attributes, and add the attribute to other models. Signed-off-by: Dan McGee --- main/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index 603d7ccc..8e705c54 100644 --- a/main/models.py +++ b/main/models.py @@ -46,7 +46,7 @@ class Donor(models.Model): class Meta: db_table = 'donors' ordering = ('name',) - get_latest_by = 'when' + get_latest_by = 'created' class Arch(models.Model): -- cgit v1.2.3-2-g168b From 911d6067bc62c02b9a4a63f4e6278d7118999869 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 11 Dec 2012 20:00:36 -0600 Subject: Update jQuery to 1.8.3 Signed-off-by: Dan McGee --- main/templatetags/cdn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'main') diff --git a/main/templatetags/cdn.py b/main/templatetags/cdn.py index f3ef9a1b..771b4426 100644 --- a/main/templatetags/cdn.py +++ b/main/templatetags/cdn.py @@ -7,7 +7,7 @@ register = template.Library() @register.simple_tag def jquery(): - version = '1.8.2' + version = '1.8.3' oncdn = getattr(settings, 'CDN_ENABLED', True) if oncdn: link = 'https://ajax.googleapis.com/ajax/libs/jquery/' \ -- cgit v1.2.3-2-g168b From 724422850020fd556e77026e54bfb56aa595eddc Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 27 Dec 2012 16:28:43 -0600 Subject: Tablesorter JS upgrade Signed-off-by: Dan McGee --- main/templatetags/cdn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'main') diff --git a/main/templatetags/cdn.py b/main/templatetags/cdn.py index 771b4426..fc63fdd8 100644 --- a/main/templatetags/cdn.py +++ b/main/templatetags/cdn.py @@ -20,7 +20,7 @@ def jquery(): @register.simple_tag def jquery_tablesorter(): - version = '2.4.5' + version = '2.7' filename = 'jquery.tablesorter-%s.min.js' % version link = staticfiles_storage.url(filename) return '' % link -- cgit v1.2.3-2-g168b From e68a5073a6e8b9473f726734e0b51fdb0a42c14b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 27 Dec 2012 17:02:40 -0600 Subject: Fix "RuntimeWarning: DateTimeField received a naive datetime" warnings When running tests, we can find old migrations that didn't use datetime objects with timezones attached. Signed-off-by: Dan McGee --- main/migrations/0024_set_initial_flag_date.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'main') diff --git a/main/migrations/0024_set_initial_flag_date.py b/main/migrations/0024_set_initial_flag_date.py index 5026f721..bd008792 100644 --- a/main/migrations/0024_set_initial_flag_date.py +++ b/main/migrations/0024_set_initial_flag_date.py @@ -1,14 +1,14 @@ # encoding: utf-8 -import datetime from south.db import db from south.v2 import DataMigration from django.db import models +from django.utils.timezone import now class Migration(DataMigration): def forwards(self, orm): orm.Package.objects.filter(needupdate=False).update(flag_date=None) - orm.Package.objects.filter(needupdate=True).update(flag_date=datetime.datetime.now()) + orm.Package.objects.filter(needupdate=True).update(flag_date=now()) def backwards(self, orm): orm.Package.objects.filter(flag_date__isnull=True).update(needupdate=False) -- cgit v1.2.3-2-g168b From bf4385a26c1b6f07bf9bdcddf7160b5eb4a71d9a Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 27 Dec 2012 21:13:56 -0600 Subject: Move the body of set_last_modified to main/utils Instead of having multiple methods, move this into our single 'created' setter method. If the 'last_modified' property is present, we now update it accordingly when saving any model with this signal attached. Signed-off-by: Dan McGee --- main/utils.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'main') diff --git a/main/utils.py b/main/utils.py index d12e5e1a..17ca386e 100644 --- a/main/utils.py +++ b/main/utils.py @@ -106,10 +106,16 @@ def retrieve_latest(sender, latest_by=None): def set_created_field(sender, **kwargs): '''This will set the 'created' field on any object to the current UTC time - if it is unset. For use as a pre_save signal handler.''' + if it is unset. + Additionally, this will set the 'last_modified' field on any object to the + current UTC time on any save of the object. + For use as a pre_save signal handler.''' obj = kwargs['instance'] + time = now() if hasattr(obj, 'created') and not obj.created: - obj.created = now() + obj.created = time + if hasattr(obj, 'last_modified'): + obj.last_modified = time def database_vendor(model, mode='read'): -- cgit v1.2.3-2-g168b From e0e2e169ec316c5d3d687e2ca211f9298f3210ee Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 27 Dec 2012 23:25:09 -0600 Subject: Remove old todo list models Signed-off-by: Dan McGee --- main/admin.py | 8 +- main/migrations/0062_remove_old_todolist_models.py | 133 +++++++++++++++++++++ main/models.py | 63 ---------- 3 files changed, 134 insertions(+), 70 deletions(-) create mode 100644 main/migrations/0062_remove_old_todolist_models.py (limited to 'main') diff --git a/main/admin.py b/main/admin.py index 741f6665..ef134187 100644 --- a/main/admin.py +++ b/main/admin.py @@ -1,5 +1,5 @@ from django.contrib import admin -from main.models import Arch, Donor, Package, Repo, Todolist +from main.models import Arch, Donor, Package, Repo class DonorAdmin(admin.ModelAdmin): list_display = ('name', 'visible', 'created') @@ -25,10 +25,6 @@ class PackageAdmin(admin.ModelAdmin): search_fields = ('pkgname', 'pkgbase', 'pkgdesc') date_hierarchy = 'build_date' -class TodolistAdmin(admin.ModelAdmin): - list_display = ('name', 'date_added', 'creator', 'description') - search_fields = ('name', 'description') - admin.site.register(Donor, DonorAdmin) @@ -36,6 +32,4 @@ admin.site.register(Package, PackageAdmin) admin.site.register(Arch, ArchAdmin) admin.site.register(Repo, RepoAdmin) -admin.site.register(Todolist, TodolistAdmin) - # vim: set ts=4 sw=4 et: diff --git a/main/migrations/0062_remove_old_todolist_models.py b/main/migrations/0062_remove_old_todolist_models.py new file mode 100644 index 00000000..46b2a4fc --- /dev/null +++ b/main/migrations/0062_remove_old_todolist_models.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + depends_on = ( + ('todolists', '0003_migrate_todolist_data'), + ) + + def forwards(self, orm): + db.delete_unique('todolist_pkgs', ['list_id', 'pkg_id']) + db.delete_table('todolists') + db.delete_table('todolist_pkgs') + + + def backwards(self, orm): + db.create_table('todolists', ( + ('description', self.gf('django.db.models.fields.TextField')()), + ('creator', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'], on_delete=models.PROTECT)), + ('date_added', self.gf('django.db.models.fields.DateTimeField')(db_index=True)), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=255)), + )) + db.send_create_signal('main', ['Todolist']) + + db.create_table('todolist_pkgs', ( + ('list', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['main.Todolist'])), + ('complete', self.gf('django.db.models.fields.BooleanField')(default=False)), + ('pkg', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['main.Package'])), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + )) + db.send_create_signal('main', ['TodolistPkg']) + + db.create_unique('todolist_pkgs', ['list_id', 'pkg_id']) + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'main.arch': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Arch', 'db_table': "'arches'"}, + 'agnostic': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'main.donor': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Donor', 'db_table': "'donors'"}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'visible': ('django.db.models.fields.BooleanField', [], {'default': 'True'}) + }, + 'main.package': { + 'Meta': {'ordering': "('pkgname',)", 'unique_together': "(('pkgname', 'repo', 'arch'),)", 'object_name': 'Package', 'db_table': "'packages'"}, + 'arch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'on_delete': 'models.PROTECT', 'to': "orm['main.Arch']"}), + 'build_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'compressed_size': ('main.fields.PositiveBigIntegerField', [], {}), + 'epoch': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}), + 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'files_last_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'flag_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'installed_size': ('main.fields.PositiveBigIntegerField', [], {}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'packager': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'on_delete': 'models.SET_NULL', 'blank': 'True'}), + 'packager_str': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pgp_signature': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'pkgdesc': ('django.db.models.fields.TextField', [], {'null': 'True'}), + 'pkgname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'repo': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'on_delete': 'models.PROTECT', 'to': "orm['main.Repo']"}), + 'url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}) + }, + 'main.packagefile': { + 'Meta': {'object_name': 'PackageFile', 'db_table': "'package_files'"}, + 'directory': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_directory': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"}) + }, + 'main.repo': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Repo', 'db_table': "'repos'"}, + 'bugs_category': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'bugs_project': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'staging': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'svn_root': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'testing': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + } + } + + complete_apps = ['main'] diff --git a/main/models.py b/main/models.py index 8e705c54..b3cc97e1 100644 --- a/main/models.py +++ b/main/models.py @@ -14,11 +14,6 @@ from .utils import cache_function, set_created_field from packages.alpm import AlpmAPI -class TodolistManager(models.Manager): - def incomplete(self): - return self.filter(todolistpkg__complete=False).order_by().distinct() - - class PackageManager(models.Manager): def flagged(self): """Used by dev dashboard.""" @@ -423,70 +418,12 @@ class PackageFile(models.Model): db_table = 'package_files' -class Todolist(models.Model): - creator = models.ForeignKey(User, on_delete=models.PROTECT) - name = models.CharField(max_length=255) - description = models.TextField() - date_added = models.DateTimeField(db_index=True) - objects = TodolistManager() - - def __unicode__(self): - return self.name - - _packages = None - - @property - def packages(self): - if not self._packages: - # select_related() does not use LEFT OUTER JOIN for nullable - # ForeignKey fields. That is why we need to explicitly list the - # ones we want. - self._packages = TodolistPkg.objects.select_related( - 'pkg__repo', 'pkg__arch').filter(list=self).order_by('pkg') - return self._packages - - @property - def package_names(self): - # depends on packages property returning a queryset - return self.packages.values_list( - 'pkg__pkgname', flat=True).order_by().distinct() - - class Meta: - db_table = 'todolists' - - def get_absolute_url(self): - return '/todo/%i/' % self.id - - def get_full_url(self, proto='https'): - '''get a URL suitable for things like email including the domain''' - domain = Site.objects.get_current().domain - return '%s://%s%s' % (proto, domain, self.get_absolute_url()) - - -class TodolistPkg(models.Model): - list = models.ForeignKey(Todolist) - pkg = models.ForeignKey(Package) - complete = models.BooleanField(default=False) - - class Meta: - db_table = 'todolist_pkgs' - unique_together = (('list','pkg'),) - - -def set_todolist_fields(sender, **kwargs): - todolist = kwargs['instance'] - if not todolist.date_added: - todolist.date_added = now() - - # connect signals needed to keep cache in line with reality from main.utils import refresh_latest from django.db.models.signals import pre_save, post_save post_save.connect(refresh_latest, sender=Package, dispatch_uid="main.models") -pre_save.connect(set_todolist_fields, sender=Todolist, - dispatch_uid="main.models") pre_save.connect(set_created_field, sender=Donor, dispatch_uid="main.models") -- cgit v1.2.3-2-g168b From 563a618e697c918c2a76c63a5217047a8d3c1489 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 28 Dec 2012 10:12:09 -0600 Subject: Move slug creation helper to main/utils Signed-off-by: Dan McGee --- main/utils.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'main') diff --git a/main/utils.py b/main/utils.py index 17ca386e..cdd4ff71 100644 --- a/main/utils.py +++ b/main/utils.py @@ -9,6 +9,7 @@ import hashlib from django.core.cache import cache from django.db import connections, router from django.utils.timezone import now +from django.template.defaultfilters import slugify CACHE_TIMEOUT = 1800 @@ -118,6 +119,20 @@ def set_created_field(sender, **kwargs): obj.last_modified = time +def find_unique_slug(model, title): + '''Attempt to find a unique slug for this model with given title.''' + existing = set(model.objects.values_list( + 'slug', flat=True).order_by().distinct()) + + suffixed = slug = slugify(title) + suffix = 0 + while suffixed in existing: + suffix += 1 + suffixed = "%s-%d" % (slug, suffix) + + return suffixed + + def database_vendor(model, mode='read'): if mode == 'read': database = router.db_for_read(model) -- cgit v1.2.3-2-g168b From 20b64e42672d185821cc584dfa4b133ee259a144 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 28 Dec 2012 14:41:18 -0600 Subject: Reduce query count when retrieving satisfiers and providers Django doesn't attach the parent object to the child objects, even when queried through the related manager. This causes up to 3 extra queries: one to retrieve the package again, and one each for arch and repo retrieval. For a package like archboot, this drops the number of necessary queries for the package page from 805 to 222 (yes, this is still too high) and cuts the time spent waiting on the database from 505ms to 262ms. Signed-off-by: Dan McGee --- main/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index b3cc97e1..ba246458 100644 --- a/main/models.py +++ b/main/models.py @@ -271,10 +271,10 @@ class Package(models.Model): arches = None # TODO: we can use list comprehension and an 'in' query to make this more effective for dep in self.depends.all(): - pkg = dep.get_best_satisfier() + pkg = dep.get_best_satisfier(self) providers = None if not pkg: - providers = dep.get_providers() + providers = dep.get_providers(self) deps.append({'dep': dep, 'pkg': pkg, 'providers': providers}) # sort the list; deptype sorting makes this tricker than expected sort_order = {'D': 0, 'O': 1, 'M': 2, 'C': 3} -- cgit v1.2.3-2-g168b From abe02e3a2a6cf4fd4b0b82c2a1dd4259a9a5debe Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 28 Dec 2012 14:52:09 -0600 Subject: Retrieve arch and repo too when calling reverse_conflicts Since we need these in the template for any details links, we might as well pull them back from the database in one query rather than three. Signed-off-by: Dan McGee --- main/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index ba246458..7ec04ad7 100644 --- a/main/models.py +++ b/main/models.py @@ -288,7 +288,7 @@ class Package(models.Model): """ Returns a list of packages with conflicts against this package. """ - pkgs = Package.objects.filter(conflicts__name=self.pkgname) + pkgs = Package.objects.normal().filter(conflicts__name=self.pkgname) if not self.arch.agnostic: # make sure we match architectures if possible pkgs = pkgs.filter(arch__in=self.applicable_arches()) -- cgit v1.2.3-2-g168b From 2fd15b4187f3ef36360ba6dcbf3c8f6cdc7517f9 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 30 Dec 2012 12:53:20 -0600 Subject: Move needs_autoescape attribute to @register.filter This is the preferred and non-deprecated way of doing this in Django 1.4+. Signed-off-by: Dan McGee --- main/templatetags/pgp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'main') diff --git a/main/templatetags/pgp.py b/main/templatetags/pgp.py index 5c9fe511..581282b4 100644 --- a/main/templatetags/pgp.py +++ b/main/templatetags/pgp.py @@ -47,7 +47,8 @@ def user_pgp_key_link(users, key_id): return pgp_key_link(key_id, matched[0].get_full_name()) return pgp_key_link(key_id) -@register.filter + +@register.filter(needs_autoescape=True) def pgp_fingerprint(key_id, autoescape=True): if not key_id: return u'' @@ -56,7 +57,6 @@ def pgp_fingerprint(key_id, autoescape=True): else: esc = lambda x: x return mark_safe(format_key(esc(key_id))) -pgp_fingerprint.needs_autoescape = True @register.assignment_tag -- cgit v1.2.3-2-g168b From 55fa2fd382241c39122a69956fa20425b1857daf Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 31 Dec 2012 00:30:17 -0600 Subject: Update the groups JSON fixture Signed-off-by: Dan McGee --- main/fixtures/groups.json | 229 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 159 insertions(+), 70 deletions(-) (limited to 'main') diff --git a/main/fixtures/groups.json b/main/fixtures/groups.json index d25c5acb..b5978b74 100644 --- a/main/fixtures/groups.json +++ b/main/fixtures/groups.json @@ -64,6 +64,115 @@ "change_signoffspecification", "packages", "signoffspecification" + ], + [ + "add_todolist", + "todolists", + "todolist" + ], + [ + "change_todolist", + "todolists", + "todolist" + ], + [ + "add_todolistpackage", + "todolists", + "todolistpackage" + ], + [ + "change_todolistpackage", + "todolists", + "todolistpackage" + ], + [ + "delete_todolistpackage", + "todolists", + "todolistpackage" + ] + ] + } + }, + { + "pk": 2, + "model": "auth.group", + "fields": { + "name": "Trusted Users", + "permissions": [ + [ + "change_package", + "main", + "package" + ], + [ + "add_todolist", + "main", + "todolist" + ], + [ + "change_todolist", + "main", + "todolist" + ], + [ + "add_todolistpkg", + "main", + "todolistpkg" + ], + [ + "change_todolistpkg", + "main", + "todolistpkg" + ], + [ + "delete_todolistpkg", + "main", + "todolistpkg" + ], + [ + "add_signoff", + "packages", + "signoff" + ], + [ + "change_signoff", + "packages", + "signoff" + ], + [ + "add_signoffspecification", + "packages", + "signoffspecification" + ], + [ + "change_signoffspecification", + "packages", + "signoffspecification" + ], + [ + "add_todolist", + "todolists", + "todolist" + ], + [ + "change_todolist", + "todolists", + "todolist" + ], + [ + "add_todolistpackage", + "todolists", + "todolistpackage" + ], + [ + "change_todolistpackage", + "todolists", + "todolistpackage" + ], + [ + "delete_todolistpackage", + "todolists", + "todolistpackage" ] ] } @@ -133,25 +242,30 @@ } }, { - "pk": 6, + "pk": 4, "model": "auth.group", "fields": { - "name": "Package Relation Maintainers", + "name": "User Admins", "permissions": [ [ - "add_packagerelation", - "packages", - "packagerelation" + "add_user", + "auth", + "user" ], [ - "change_packagerelation", - "packages", - "packagerelation" + "change_user", + "auth", + "user" ], [ - "delete_packagerelation", - "packages", - "packagerelation" + "add_userprofile", + "devel", + "userprofile" + ], + [ + "change_userprofile", + "devel", + "userprofile" ] ] } @@ -312,6 +426,21 @@ "releng", "module" ], + [ + "add_release", + "releng", + "release" + ], + [ + "change_release", + "releng", + "release" + ], + [ + "delete_release", + "releng", + "release" + ], [ "add_source", "releng", @@ -346,89 +475,49 @@ } }, { - "pk": 2, + "pk": 6, "model": "auth.group", "fields": { - "name": "Trusted Users", + "name": "Package Relation Maintainers", "permissions": [ [ - "change_package", - "main", - "package" - ], - [ - "add_todolist", - "main", - "todolist" - ], - [ - "change_todolist", - "main", - "todolist" - ], - [ - "add_todolistpkg", - "main", - "todolistpkg" - ], - [ - "change_todolistpkg", - "main", - "todolistpkg" - ], - [ - "delete_todolistpkg", - "main", - "todolistpkg" - ], - [ - "add_signoff", - "packages", - "signoff" - ], - [ - "change_signoff", + "add_packagerelation", "packages", - "signoff" + "packagerelation" ], [ - "add_signoffspecification", + "change_packagerelation", "packages", - "signoffspecification" + "packagerelation" ], [ - "change_signoffspecification", + "delete_packagerelation", "packages", - "signoffspecification" + "packagerelation" ] ] } }, { - "pk": 4, + "pk": 8, "model": "auth.group", "fields": { - "name": "User Admins", + "name": "Download Page Releases", "permissions": [ [ - "add_user", - "auth", - "user" - ], - [ - "change_user", - "auth", - "user" + "add_release", + "releng", + "release" ], [ - "add_userprofile", - "devel", - "userprofile" + "change_release", + "releng", + "release" ], [ - "change_userprofile", - "devel", - "userprofile" + "delete_release", + "releng", + "release" ] ] } -- cgit v1.2.3-2-g168b From 7620eff2f2f77b86f07cc6224bee54685cc1b3ab Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 31 Dec 2012 00:31:21 -0600 Subject: Remove old todolist permissions Signed-off-by: Dan McGee --- main/fixtures/groups.json | 50 ----------------------------------------------- 1 file changed, 50 deletions(-) (limited to 'main') diff --git a/main/fixtures/groups.json b/main/fixtures/groups.json index b5978b74..134b98b3 100644 --- a/main/fixtures/groups.json +++ b/main/fixtures/groups.json @@ -10,31 +10,6 @@ "main", "package" ], - [ - "add_todolist", - "main", - "todolist" - ], - [ - "change_todolist", - "main", - "todolist" - ], - [ - "add_todolistpkg", - "main", - "todolistpkg" - ], - [ - "change_todolistpkg", - "main", - "todolistpkg" - ], - [ - "delete_todolistpkg", - "main", - "todolistpkg" - ], [ "add_news", "news", @@ -104,31 +79,6 @@ "main", "package" ], - [ - "add_todolist", - "main", - "todolist" - ], - [ - "change_todolist", - "main", - "todolist" - ], - [ - "add_todolistpkg", - "main", - "todolistpkg" - ], - [ - "change_todolistpkg", - "main", - "todolistpkg" - ], - [ - "delete_todolistpkg", - "main", - "todolistpkg" - ], [ "add_signoff", "packages", -- cgit v1.2.3-2-g168b From 04f23a040a839f4989fdc83afe0f5ad4f72224be Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 31 Dec 2012 10:17:22 -0600 Subject: Add 'created' field to packages model This will be used to eventually implement the UI side of FS#13441, but to do that, we first need the data. Signed-off-by: Dan McGee --- .../0063_auto__add_field_package_created.py | 116 +++++++++++++++++++++ main/models.py | 3 + 2 files changed, 119 insertions(+) create mode 100644 main/migrations/0063_auto__add_field_package_created.py (limited to 'main') diff --git a/main/migrations/0063_auto__add_field_package_created.py b/main/migrations/0063_auto__add_field_package_created.py new file mode 100644 index 00000000..e5a990c3 --- /dev/null +++ b/main/migrations/0063_auto__add_field_package_created.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +import datetime +from pytz import utc +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + old_date = datetime.datetime(2000, 1, 1) + old_date = old_date.replace(tzinfo=utc) + db.add_column('packages', 'created', + self.gf('django.db.models.fields.DateTimeField')(default=old_date), keep_default=False) + + + def backwards(self, orm): + db.delete_column('packages', 'created') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'main.arch': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Arch', 'db_table': "'arches'"}, + 'agnostic': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'main.donor': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Donor', 'db_table': "'donors'"}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'visible': ('django.db.models.fields.BooleanField', [], {'default': 'True'}) + }, + 'main.package': { + 'Meta': {'ordering': "('pkgname',)", 'unique_together': "(('pkgname', 'repo', 'arch'),)", 'object_name': 'Package', 'db_table': "'packages'"}, + 'arch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'on_delete': 'models.PROTECT', 'to': "orm['main.Arch']"}), + 'build_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'compressed_size': ('main.fields.PositiveBigIntegerField', [], {}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'epoch': ('django.db.models.fields.PositiveIntegerField', [], {'default': '0'}), + 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'files_last_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'flag_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'installed_size': ('main.fields.PositiveBigIntegerField', [], {}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'packager': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'on_delete': 'models.SET_NULL', 'blank': 'True'}), + 'packager_str': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pgp_signature': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'pkgdesc': ('django.db.models.fields.TextField', [], {'null': 'True'}), + 'pkgname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'repo': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'on_delete': 'models.PROTECT', 'to': "orm['main.Repo']"}), + 'url': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}) + }, + 'main.packagefile': { + 'Meta': {'object_name': 'PackageFile', 'db_table': "'package_files'"}, + 'directory': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'filename': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_directory': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"}) + }, + 'main.repo': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Repo', 'db_table': "'repos'"}, + 'bugs_category': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'bugs_project': ('django.db.models.fields.SmallIntegerField', [], {'default': '1'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'staging': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'svn_root': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'testing': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + } + } + + complete_apps = ['main'] diff --git a/main/models.py b/main/models.py index 7ec04ad7..39bc555e 100644 --- a/main/models.py +++ b/main/models.py @@ -103,6 +103,7 @@ class Package(models.Model): build_date = models.DateTimeField(null=True) last_update = models.DateTimeField(db_index=True) files_last_update = models.DateTimeField(null=True, blank=True) + created = models.DateTimeField() packager_str = models.CharField(max_length=255) packager = models.ForeignKey(User, null=True, blank=True, on_delete=models.SET_NULL) @@ -424,6 +425,8 @@ from django.db.models.signals import pre_save, post_save post_save.connect(refresh_latest, sender=Package, dispatch_uid="main.models") +# note: reporead sets the 'created' field on Package objects, so no signal +# listener is set up here to do so pre_save.connect(set_created_field, sender=Donor, dispatch_uid="main.models") -- cgit v1.2.3-2-g168b From c37fe107282f1aa4925d6c3eef9b7c1598ab4aa1 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 31 Dec 2012 10:24:09 -0600 Subject: Minor coding style tweaks Signed-off-by: Dan McGee --- main/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index 39bc555e..7155d360 100644 --- a/main/models.py +++ b/main/models.py @@ -270,7 +270,8 @@ class Package(models.Model): """ deps = [] arches = None - # TODO: we can use list comprehension and an 'in' query to make this more effective + # TODO: we can use list comprehension and an 'in' query to make this + # more effective for dep in self.depends.all(): pkg = dep.get_best_satisfier(self) providers = None @@ -279,6 +280,7 @@ class Package(models.Model): deps.append({'dep': dep, 'pkg': pkg, 'providers': providers}) # sort the list; deptype sorting makes this tricker than expected sort_order = {'D': 0, 'O': 1, 'M': 2, 'C': 3} + def sort_key(val): dep = val['dep'] return (sort_order.get(dep.deptype, 1000), dep.name) -- cgit v1.2.3-2-g168b From 3a6398f42d04ea6a677bf7b6d5115175e9011432 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 15 Jan 2013 21:27:37 -0600 Subject: Add new AlwaysCommitMiddleware to the stack The reason for this is documented in the middleware itself. Without this, pgbouncer is of little use to us since it has to throw away every connection we try to route through it because of unclean disconnects. In theory, with the switch to using pgbouncer for all WSGI originating connections and adding this middleware, we should see a notable decrease in connection time to the database. Signed-off-by: Dan McGee --- main/middleware.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 main/middleware.py (limited to 'main') diff --git a/main/middleware.py b/main/middleware.py new file mode 100644 index 00000000..a698b13c --- /dev/null +++ b/main/middleware.py @@ -0,0 +1,40 @@ +from django.core.exceptions import MiddlewareNotUsed +from django.db import connections + + +class AlwaysCommitMiddleware(object): + """ + Ensure we always commit any possibly open transaction so we leave the + database in a clean state. Without this, pgbouncer et al. always gives + error messages like this for every single request: + + LOG S-0x1accfd0: db/user@unix:5432 new connection to server + LOG C-0x1aaf620: db/user@unix:6432 closing because: client close request (age=0) + LOG S-0x1accfd0: db/user@unix:5432 closing because: unclean server (age=0) + + We only let this middleware apply for PostgreSQL backends; other databases + don't really require connection pooling and thus the reason for this + middleware's use is non-existent. + + The best location of this in your middleware stack is likely the top, as + you want to ensure it happens after any and all database activity has + completed. + """ + def __init__(self): + for conn in connections.all(): + if conn.vendor == 'postgresql': + return + raise MiddlewareNotUsed() + + def process_response(self, request, response): + """Commits any potentially open transactions at the underlying + PostgreSQL database connection level.""" + for conn in connections.all(): + if conn.vendor != 'postgresql': + continue + db_conn = getattr(conn, 'connection', None) + if db_conn is not None: + db_conn.commit() + return response + +# vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From f106379b5382b2b82aa56466c8d3acaae58327a9 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 15 Jan 2013 21:29:30 -0600 Subject: Clean up and make several migrations modern This moves most migrations to the v2 format that have been presenting some issues. One missing depends_on relationship has been added, and we allow an index to not be dropped if it does not exist due to the shittyness in sqlite3 actually keeping indexes across DDL on that table. Signed-off-by: Dan McGee --- main/migrations/0002_make_maintainer_nullable.py | 23 +++++++---------------- main/migrations/0003_migrate_maintainer.py | 8 +++----- main/migrations/0005_fix_empty_url_pkgdesc.py | 10 +++------- main/migrations/0013_mark_repos_testing.py | 6 +++--- main/migrations/0055_unique_package_in_repo.py | 7 ++++++- 5 files changed, 22 insertions(+), 32 deletions(-) (limited to 'main') diff --git a/main/migrations/0002_make_maintainer_nullable.py b/main/migrations/0002_make_maintainer_nullable.py index 138b103b..675635df 100644 --- a/main/migrations/0002_make_maintainer_nullable.py +++ b/main/migrations/0002_make_maintainer_nullable.py @@ -1,26 +1,17 @@ - +# encoding: utf-8 from south.db import db +from south.v2 import SchemaMigration from django.db import models -from main.models import * -class Migration: - +class Migration(SchemaMigration): + def forwards(self, orm): - - # Changing field 'Package.maintainer' - # (to signature: django.db.models.fields.related.ForeignKey(null=True, to=orm['auth.User'])) db.alter_column('packages', 'maintainer_id', orm['main.package:maintainer']) - - - + def backwards(self, orm): - - # Changing field 'Package.maintainer' - # (to signature: django.db.models.fields.related.ForeignKey(to=orm['auth.User'])) db.alter_column('packages', 'maintainer_id', orm['main.package:maintainer']) - - - + + models = { 'auth.group': { 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), diff --git a/main/migrations/0003_migrate_maintainer.py b/main/migrations/0003_migrate_maintainer.py index a3a4793f..4169a5c9 100644 --- a/main/migrations/0003_migrate_maintainer.py +++ b/main/migrations/0003_migrate_maintainer.py @@ -1,11 +1,9 @@ - +# -*- coding: utf-8 -*- from south.db import db +from south.v2 import DataMigration from django.db import models -from main.models import * - -class Migration: - no_dry_run = True +class Migration(DataMigration): def forwards(self, orm): orm.Package.objects.filter(maintainer=0).update(maintainer=None) diff --git a/main/migrations/0005_fix_empty_url_pkgdesc.py b/main/migrations/0005_fix_empty_url_pkgdesc.py index c7cc1d8c..54658c17 100644 --- a/main/migrations/0005_fix_empty_url_pkgdesc.py +++ b/main/migrations/0005_fix_empty_url_pkgdesc.py @@ -1,14 +1,11 @@ - +# -*- coding: utf-8 -*- from south.db import db +from south.v2 import DataMigration from django.db import models -from main.models import * -class Migration: +class Migration(DataMigration): - no_dry_run = True - def forwards(self, orm): - "Write your forwards migration here" for p in orm.Package.objects.filter(pkgdesc=''): p.pkgdesc = None p.save() @@ -24,7 +21,6 @@ class Migration: def backwards(self, orm): - "Write your backwards migration here" for p in orm.Package.objects.filter(pkgdesc=None): p.pkgdesc = '' p.save() diff --git a/main/migrations/0013_mark_repos_testing.py b/main/migrations/0013_mark_repos_testing.py index 617a3ab8..e50010b2 100644 --- a/main/migrations/0013_mark_repos_testing.py +++ b/main/migrations/0013_mark_repos_testing.py @@ -1,9 +1,9 @@ +# -*- coding: utf-8 -*- from south.db import db +from south.v2 import DataMigration from django.db import models -from main.models import * -class Migration: - no_dry_run = True +class Migration(DataMigration): def forwards(self, orm): orm.Repo.objects.filter(name__endswith="Testing").update(testing=True) diff --git a/main/migrations/0055_unique_package_in_repo.py b/main/migrations/0055_unique_package_in_repo.py index 36cc7193..9ae33719 100644 --- a/main/migrations/0055_unique_package_in_repo.py +++ b/main/migrations/0055_unique_package_in_repo.py @@ -2,11 +2,16 @@ from south.db import db from south.v2 import SchemaMigration from django.db import models +from django.db.utils import DatabaseError class Migration(SchemaMigration): def forwards(self, orm): - db.delete_index('packages', ['pkgname']) + try: + db.delete_index('packages', ['pkgname']) + except DatabaseError as e: + if not 'no such index' in str(e): + raise e db.create_unique('packages', ['pkgname', 'repo_id', 'arch_id']) def backwards(self, orm): -- cgit v1.2.3-2-g168b From 0b930fd92140858f4ad21e593feb057996af9b95 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 16 Jan 2013 00:36:17 -0600 Subject: Convert all usages of flag icons to new sprite This uses a new template tag to avoid repeating construction of the necessary HTML element all over the place. The site should look exactly as it did before, except now you don't have to download 20+ images to see some pages. Signed-off-by: Dan McGee --- main/templatetags/flags.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 main/templatetags/flags.py (limited to 'main') diff --git a/main/templatetags/flags.py b/main/templatetags/flags.py new file mode 100644 index 00000000..22f524ca --- /dev/null +++ b/main/templatetags/flags.py @@ -0,0 +1,13 @@ +from django import template + +register = template.Library() + + +@register.simple_tag +def country_flag(country): + if not country: + return '' + return ' ' % ( + country.code.lower(), country.name) + +# vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From 1b1b516bd823d807ea81e62fe14fc92c18d0b89d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 20 Jan 2013 14:59:30 -0600 Subject: Query performance enhancements in get_requiredby() For packages with particularly long lists of provides (e.g. perl), the query was getting a bit out of control with the list of names passed in. However, changing it to simply do a subquery resulted in some really poor planning by PostgreSQL. Doing this as a custom 'WHERE' clause utilizing the 'UNION ALL' SQL operator works very well. Signed-off-by: Dan McGee --- main/models.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index 7155d360..88f0ecd1 100644 --- a/main/models.py +++ b/main/models.py @@ -190,12 +190,13 @@ class Package(models.Model): category as this package if that check makes sense. """ from packages.models import Depend - provides = self.provides.all() - provide_names = {provide.name for provide in provides} - provide_names.add(self.pkgname) + name_clause = '''packages_depend.name IN ( + SELECT %s UNION ALL + SELECT z.name FROM packages_provision z WHERE z.pkg_id = %s + )''' requiredby = Depend.objects.select_related('pkg', - 'pkg__arch', 'pkg__repo').filter( - name__in=provide_names).order_by( + 'pkg__arch', 'pkg__repo').extra( + where=[name_clause], params=[self.pkgname, self.id]).order_by( 'pkg__pkgname', 'pkg__arch__name', 'pkg__repo__name') if not self.arch.agnostic: # make sure we match architectures if possible -- cgit v1.2.3-2-g168b From 45108ea4975419a88c2bb10ed7f3f90d6085d852 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 20 Jan 2013 15:09:18 -0600 Subject: Remove AlwaysCommitMiddleware Let's just go with the Django database option for PostreSQL autocommit mode instead. Signed-off-by: Dan McGee --- main/middleware.py | 40 ---------------------------------------- 1 file changed, 40 deletions(-) delete mode 100644 main/middleware.py (limited to 'main') diff --git a/main/middleware.py b/main/middleware.py deleted file mode 100644 index a698b13c..00000000 --- a/main/middleware.py +++ /dev/null @@ -1,40 +0,0 @@ -from django.core.exceptions import MiddlewareNotUsed -from django.db import connections - - -class AlwaysCommitMiddleware(object): - """ - Ensure we always commit any possibly open transaction so we leave the - database in a clean state. Without this, pgbouncer et al. always gives - error messages like this for every single request: - - LOG S-0x1accfd0: db/user@unix:5432 new connection to server - LOG C-0x1aaf620: db/user@unix:6432 closing because: client close request (age=0) - LOG S-0x1accfd0: db/user@unix:5432 closing because: unclean server (age=0) - - We only let this middleware apply for PostgreSQL backends; other databases - don't really require connection pooling and thus the reason for this - middleware's use is non-existent. - - The best location of this in your middleware stack is likely the top, as - you want to ensure it happens after any and all database activity has - completed. - """ - def __init__(self): - for conn in connections.all(): - if conn.vendor == 'postgresql': - return - raise MiddlewareNotUsed() - - def process_response(self, request, response): - """Commits any potentially open transactions at the underlying - PostgreSQL database connection level.""" - for conn in connections.all(): - if conn.vendor != 'postgresql': - continue - db_conn = getattr(conn, 'connection', None) - if db_conn is not None: - db_conn.commit() - return response - -# vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From 53484c45ea82a5afa8bf167f978f657b866d4c93 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 20 Jan 2013 15:27:16 -0600 Subject: Fix error in get_requiredby() when checking provides The query refactor in commit 1b1b516bd removed a queryset I didn't realize was getting used elsewhere in the function. Signed-off-by: Dan McGee --- main/models.py | 1 + 1 file changed, 1 insertion(+) (limited to 'main') diff --git a/main/models.py b/main/models.py index 88f0ecd1..40466d65 100644 --- a/main/models.py +++ b/main/models.py @@ -207,6 +207,7 @@ class Package(models.Model): # version comparison operators they may specify alpm = AlpmAPI() if alpm.available: + provides = self.provides.all() new_rqd = [] for dep in requiredby: if not dep.comparison or not dep.version: -- cgit v1.2.3-2-g168b From d63a800348f81823f157ec9ed03f9985308c3ea3 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 3 Feb 2013 14:40:41 -0600 Subject: Update user_pgp_key_link template tag to use DeveloperKey model The first of several small updates to use the new data we have available. Signed-off-by: Dan McGee --- main/templatetags/pgp.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'main') diff --git a/main/templatetags/pgp.py b/main/templatetags/pgp.py index 581282b4..afad9df2 100644 --- a/main/templatetags/pgp.py +++ b/main/templatetags/pgp.py @@ -3,8 +3,11 @@ from django.conf import settings from django.utils.html import conditional_escape from django.utils.safestring import mark_safe +from devel.models import DeveloperKey + register = template.Library() + def format_key(key_id): if len(key_id) in (8, 20): return u'0x%s' % key_id @@ -40,12 +43,14 @@ def pgp_key_link(key_id, link_text=None): return '%s' % values @register.simple_tag -def user_pgp_key_link(users, key_id): - matched = [user for user in users if user.userprofile.pgp_key and - user.userprofile.pgp_key[-16:] == key_id[-16:]] - if matched and len(matched) == 1: - return pgp_key_link(key_id, matched[0].get_full_name()) - return pgp_key_link(key_id) +def user_pgp_key_link(key_id): + normalized = key_id[-16:] + try: + matching_key = DeveloperKey.objects.select_related( + 'owner').get(key=normalized, owner_id__isnull=False) + except DeveloperKey.DoesNotExist: + return pgp_key_link(key_id) + return pgp_key_link(key_id, matching_key.owner.get_full_name()) @register.filter(needs_autoescape=True) -- cgit v1.2.3-2-g168b From 508d06af810c787b2644331444279407ccfa27af Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 3 Feb 2013 15:08:21 -0600 Subject: Use DeveloperKey model on package page and reports This introduces the new model to the package page so subkey signings show up as attributed to the original developer. We also teach the mismatched signatures report to recognize all keys and subkeys of a given developer, cutting down on some of the bogus results. Signed-off-by: Dan McGee --- main/models.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index 40466d65..53a24ffc 100644 --- a/main/models.py +++ b/main/models.py @@ -11,6 +11,7 @@ from django.utils.timezone import now from .fields import PositiveBigIntegerField from .utils import cache_function, set_created_field +from devel.models import DeveloperKey from packages.alpm import AlpmAPI @@ -153,8 +154,9 @@ class Package(models.Model): sig = self.signature if sig and sig.key_id: try: - user = User.objects.get( - userprofile__pgp_key__endswith=sig.key_id) + matching_key = DeveloperKey.objects.select_related( + 'owner').get(key=sig.key_id, owner_id__isnull=False) + user = matching_key.owner except User.DoesNotExist: user = None return user -- cgit v1.2.3-2-g168b From cfa2798880eccda63a3ed4d3eddadeb01f5065d2 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 4 Feb 2013 17:01:58 -0600 Subject: Update exception to DeveloperKey.DoesNotExist We aren't looking up users; we are looking up developer keys. Signed-off-by: Dan McGee --- main/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index 53a24ffc..46fd3a63 100644 --- a/main/models.py +++ b/main/models.py @@ -157,7 +157,7 @@ class Package(models.Model): matching_key = DeveloperKey.objects.select_related( 'owner').get(key=sig.key_id, owner_id__isnull=False) user = matching_key.owner - except User.DoesNotExist: + except DeveloperKey.DoesNotExist: user = None return user return None -- cgit v1.2.3-2-g168b From 8d79a1ea84756b016fb76d940e95a8885d014dae Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 8 Feb 2013 21:03:52 -0600 Subject: Minify static files when running collectstatic This doesn't do any super optimizations, but does run the very basic cssmin and jsmin Python tools over the static resources we serve up. Signed-off-by: Dan McGee --- main/storage.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 main/storage.py (limited to 'main') diff --git a/main/storage.py b/main/storage.py new file mode 100644 index 00000000..62e94ef7 --- /dev/null +++ b/main/storage.py @@ -0,0 +1,36 @@ +import cssmin +import jsmin + +from django.contrib.staticfiles.storage import CachedStaticFilesStorage +from django.core.files.base import ContentFile +from django.utils.encoding import smart_str + + +class MinifiedStaticFilesStorage(CachedStaticFilesStorage): + """ + A static file system storage backend which minifies the hashed + copies of the files it saves. It currently knows how to process + CSS and JS files. Files containing '.min' anywhere in the filename + are skipped as they are already assumed minified. + """ + minifiers = ( + ('.css', cssmin.cssmin), + ('.js', jsmin.jsmin), + ) + + def post_process(self, paths, dry_run=False, **options): + for original_path, processed_path, processed in super( + MinifiedStaticFilesStorage, self).post_process( + paths, dry_run, **options): + for ext, func in self.minifiers: + if '.min' in original_path: + continue + if original_path.endswith(ext): + with self._open(processed_path) as processed_file: + minified = func(processed_file.read()) + minified_file = ContentFile(smart_str(minified)) + self.delete(processed_path) + self._save(processed_path, minified_file) + processed = True + + yield original_path, processed_path, processed -- cgit v1.2.3-2-g168b From 271d1babbf8038e17d9dc5cfc3cd659463848400 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 20 Jan 2013 12:43:33 -0600 Subject: Revert "Reduce query count when retrieving satisfiers and providers" This reverts commit 20b64e42672d185821cc584dfa4b133ee259a144. Django 1.5 fixed this issue and now parent objects are automatically attached to their children when queries go through the related manager. See "Caching of related model instances" in the release notes. Signed-off-by: Dan McGee --- main/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index 46fd3a63..da9d8b6e 100644 --- a/main/models.py +++ b/main/models.py @@ -277,10 +277,10 @@ class Package(models.Model): # TODO: we can use list comprehension and an 'in' query to make this # more effective for dep in self.depends.all(): - pkg = dep.get_best_satisfier(self) + pkg = dep.get_best_satisfier() providers = None if not pkg: - providers = dep.get_providers(self) + providers = dep.get_providers() deps.append({'dep': dep, 'pkg': pkg, 'providers': providers}) # sort the list; deptype sorting makes this tricker than expected sort_order = {'D': 0, 'O': 1, 'M': 2, 'C': 3} -- cgit v1.2.3-2-g168b From 8524a6057c4b99a620850494a22eb3d1f56bee68 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 20 Jan 2013 14:15:53 -0600 Subject: Change caching strategy on package.applicable_arches Rather than use the Django cache for this (aka memcached), just do it on a per-object instantiation basis. This means no external services calls except the first time to the database, which should be quite a bit faster. Signed-off-by: Dan McGee --- main/models.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index da9d8b6e..ec5c2bd6 100644 --- a/main/models.py +++ b/main/models.py @@ -177,12 +177,15 @@ class Package(models.Model): def maintainers(self, maintainers): self._maintainers = maintainers - @cache_function(1800) + _applicable_arches = None + def applicable_arches(self): '''The list of (this arch) + (available agnostic arches).''' - arches = set(Arch.objects.filter(agnostic=True)) - arches.add(self.arch) - return list(arches) + if self._applicable_arches is None: + arches = set(Arch.objects.filter(agnostic=True)) + arches.add(self.arch) + self._applicable_arches = list(arches) + return self._applicable_arches @cache_function(119) def get_requiredby(self): -- cgit v1.2.3-2-g168b From d5a644696466e443a590317ac2f892ac279665ab Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 27 Feb 2013 08:40:23 -0600 Subject: Remove caching on reverse_conflicts We started seeing this in production with memcached usage and the upgrade to Django 1.5: PicklingError: Can't pickle : attribute lookup devel.models.UserProfile_allowed_repos failed Without having time to investigate further, something changed, likely due to the whole user profile/configurable user model shenanigans done in Django 1.5. For now, simply don't cache this attribute to work around the problem. Signed-off-by: Dan McGee --- main/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index ec5c2bd6..9410e135 100644 --- a/main/models.py +++ b/main/models.py @@ -293,7 +293,7 @@ class Package(models.Model): return (sort_order.get(dep.deptype, 1000), dep.name) return sorted(deps, key=sort_key) - @cache_function(123) + #@cache_function(123) def reverse_conflicts(self): """ Returns a list of packages with conflicts against this package. -- cgit v1.2.3-2-g168b From d98ab844a03b0579051cece998e7c18a048c21ab Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 27 Feb 2013 09:00:17 -0600 Subject: Disable more @cache_function usage Seems the Django caching layer has changed quite a bit and is doing all sorts of funky s**t at this point. Yay for errors! Signed-off-by: Dan McGee --- main/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'main') diff --git a/main/models.py b/main/models.py index 9410e135..a561f4f6 100644 --- a/main/models.py +++ b/main/models.py @@ -187,7 +187,7 @@ class Package(models.Model): self._applicable_arches = list(arches) return self._applicable_arches - @cache_function(119) + #@cache_function(119) def get_requiredby(self): """ Returns a list of package objects. An attempt will be made to keep this @@ -265,7 +265,7 @@ class Package(models.Model): trimmed.append(dep) return trimmed - @cache_function(121) + #@cache_function(121) def get_depends(self): """ Returns a list of dicts. Each dict contains ('dep', 'pkg', and -- cgit v1.2.3-2-g168b