From 55179a4f9e8b80d515bae7032af8aefc33ae0192 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 16 Jan 2013 16:07:26 -0600 Subject: reporead: remove batched_bulk_create Now that Django 1.5 is out and realized SQLite3 only allows for 999 parameters per SQL call, we don't need to manually batch things up anymore and can let the underlying bulk_create code do it for us. This basically reverts commit 88ee61a39ac3. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 32 +++++--------------------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index ab0efeed..ccac55f2 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -32,7 +32,6 @@ from django.utils.timezone import now from devel.utils import UserFinder from main.models import Arch, Package, PackageFile, Repo -from main.utils import database_vendor from packages.models import Depend, Conflict, Provision, Replacement, Update from packages.utils import parse_version @@ -182,27 +181,6 @@ def create_related(model, package, rel_str, equals_only=False): return related -def batched_bulk_create(model, all_objects): - # for short lists, just bulk_create as we should be fine - if len(all_objects) < 20: - return model.objects.bulk_create(all_objects) - - if database_vendor(model, mode='write') == 'sqlite': - # 999 max variables in each SQL statement - incr = 999 // len(model._meta.fields) - else: - incr = 1000 - - def chunks(): - offset = 0 - while offset < len(all_objects): - yield all_objects[offset:offset + incr] - offset += incr - - for items in chunks(): - model.objects.bulk_create(items) - - def create_multivalued(dbpkg, repopkg, db_attr, repo_attr): '''Populate the simplest of multivalued attributes. These are those that only deal with a 'name' attribute, such as licenses, groups, etc. The input @@ -256,20 +234,20 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): deps += [create_depend(dbpkg, y, 'O') for y in repopkg.optdepends] deps += [create_depend(dbpkg, y, 'M') for y in repopkg.makedepends] deps += [create_depend(dbpkg, y, 'C') for y in repopkg.checkdepends] - batched_bulk_create(Depend, deps) + Depend.objects.bulk_create(deps) dbpkg.conflicts.all().delete() conflicts = [create_related(Conflict, dbpkg, y) for y in repopkg.conflicts] - batched_bulk_create(Conflict, conflicts) + Conflict.objects.bulk_create(conflicts) dbpkg.provides.all().delete() provides = [create_related(Provision, dbpkg, y, equals_only=True) for y in repopkg.provides] - batched_bulk_create(Provision, provides) + Provision.objects.bulk_create(provides) dbpkg.replaces.all().delete() replaces = [create_related(Replacement, dbpkg, y) for y in repopkg.replaces] - batched_bulk_create(Replacement, replaces) + Replacement.objects.bulk_create(replaces) create_multivalued(dbpkg, repopkg, 'groups', 'groups') create_multivalued(dbpkg, repopkg, 'licenses', 'license') @@ -319,7 +297,7 @@ def populate_files(dbpkg, repopkg, force=False): directory=dirname, filename=filename) pkg_files.append(pkgfile) - batched_bulk_create(PackageFile, pkg_files) + PackageFile.objects.bulk_create(pkg_files) dbpkg.files_last_update = now() dbpkg.save() -- cgit v1.1-4-g5e80 From 10462425f989dc74653179f0845978a6b5e30045 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 16 Jan 2013 16:10:24 -0600 Subject: Remove {% load url from future %} from templates This is now the default in Django 1.5. Signed-off-by: Dan McGee --- templates/base.html | 2 +- templates/packages/removed.html | 1 - templates/public/download.html | 1 - templates/public/index.html | 1 - templates/public/keys.html | 1 - templates/releng/add.html | 1 - templates/releng/iso_overview.html | 1 - templates/releng/release_detail.html | 1 - templates/releng/release_list.html | 1 - templates/releng/result_list.html | 1 - templates/releng/result_section.html | 1 - templates/releng/results.html | 1 - templates/releng/thanks.html | 1 - templates/visualize/index.html | 1 - 14 files changed, 1 insertion(+), 14 deletions(-) diff --git a/templates/base.html b/templates/base.html index cc507fbf..15bb7e28 100644 --- a/templates/base.html +++ b/templates/base.html @@ -1,4 +1,4 @@ -{% load url from future %}{% load static from staticfiles %} +{% load static from staticfiles %} diff --git a/templates/packages/removed.html b/templates/packages/removed.html index ea20ce80..f188b6db 100644 --- a/templates/packages/removed.html +++ b/templates/packages/removed.html @@ -1,5 +1,4 @@ {% extends "base.html" %} -{% load url from future %} {% load package_extras %} {% block title %}Arch Linux - Not Available - {{ name }} {{ version }} ({{ arch.name }}){% endblock %} diff --git a/templates/public/download.html b/templates/public/download.html index c68cf66b..7936efb5 100644 --- a/templates/public/download.html +++ b/templates/public/download.html @@ -1,6 +1,5 @@ {% extends "base.html" %} {% load cache %} -{% load url from future %} {% load static from staticfiles %} {% load flags %} diff --git a/templates/public/index.html b/templates/public/index.html index 8926a061..3f88c183 100644 --- a/templates/public/index.html +++ b/templates/public/index.html @@ -1,6 +1,5 @@ {% extends "base.html" %} {% load cache %} -{% load url from future %} {% load static from staticfiles %} {% block head %} diff --git a/templates/public/keys.html b/templates/public/keys.html index ad2dd19d..91b53075 100644 --- a/templates/public/keys.html +++ b/templates/public/keys.html @@ -1,6 +1,5 @@ {% extends "base.html" %} {% load static from staticfiles %} -{% load url from future %} {% load pgp %} {% block title %}Arch Linux - Master Signing Keys{% endblock %} diff --git a/templates/releng/add.html b/templates/releng/add.html index ed02984e..d060395d 100644 --- a/templates/releng/add.html +++ b/templates/releng/add.html @@ -1,5 +1,4 @@ {% extends "base.html" %} -{% load url from future %} {% block title %}Arch Linux - Test Result Entry{% endblock %} diff --git a/templates/releng/iso_overview.html b/templates/releng/iso_overview.html index 23ef0135..196f0c0a 100644 --- a/templates/releng/iso_overview.html +++ b/templates/releng/iso_overview.html @@ -1,6 +1,5 @@ {% extends "base.html" %} {% load static from staticfiles %} -{% load url from future %} {% block content %}
diff --git a/templates/releng/release_detail.html b/templates/releng/release_detail.html index 01c0319e..09507536 100644 --- a/templates/releng/release_detail.html +++ b/templates/releng/release_detail.html @@ -1,5 +1,4 @@ {% extends "base.html" %} -{% load url from future %} {% block title %}Arch Linux - Release: {{ release.version }}{% endblock %} diff --git a/templates/releng/release_list.html b/templates/releng/release_list.html index ef53a93d..5f248264 100644 --- a/templates/releng/release_list.html +++ b/templates/releng/release_list.html @@ -1,5 +1,4 @@ {% extends "base.html" %} -{% load url from future %} {% load static from staticfiles %} {% block title %}Arch Linux - Releases{% endblock %} diff --git a/templates/releng/result_list.html b/templates/releng/result_list.html index be5783b3..12bf39db 100644 --- a/templates/releng/result_list.html +++ b/templates/releng/result_list.html @@ -1,6 +1,5 @@ {% extends "base.html" %} {% load static from staticfiles %} -{% load url from future %} {% block content %}
diff --git a/templates/releng/result_section.html b/templates/releng/result_section.html index ae951ebe..5e0b6f62 100644 --- a/templates/releng/result_section.html +++ b/templates/releng/result_section.html @@ -1,4 +1,3 @@ -{% load url from future %} {% if option.is_rollback %}Rollback: {% endif %}{{ option.name|title }} Last Success diff --git a/templates/releng/results.html b/templates/releng/results.html index c19b42a6..59d8351d 100644 --- a/templates/releng/results.html +++ b/templates/releng/results.html @@ -1,5 +1,4 @@ {% extends "base.html" %} -{% load url from future %} {% block title %}Arch Linux - Release Engineering Testbuild Results{% endblock %} diff --git a/templates/releng/thanks.html b/templates/releng/thanks.html index 66d65a5c..2462dafd 100644 --- a/templates/releng/thanks.html +++ b/templates/releng/thanks.html @@ -1,5 +1,4 @@ {% extends "base.html" %} -{% load url from future %} {% block title %}Arch Linux - Feedback - Thanks!{% endblock %} diff --git a/templates/visualize/index.html b/templates/visualize/index.html index 1e932d9b..2b79d601 100644 --- a/templates/visualize/index.html +++ b/templates/visualize/index.html @@ -1,6 +1,5 @@ {% extends "base.html" %} {% load static from staticfiles %} -{% load url from future %} {% block title %}Arch Linux - Visualizations{% endblock %} -- cgit v1.1-4-g5e80 From 271d1babbf8038e17d9dc5cfc3cd659463848400 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 20 Jan 2013 12:43:33 -0600 Subject: Revert "Reduce query count when retrieving satisfiers and providers" This reverts commit 20b64e42672d185821cc584dfa4b133ee259a144. Django 1.5 fixed this issue and now parent objects are automatically attached to their children when queries go through the related manager. See "Caching of related model instances" in the release notes. Signed-off-by: Dan McGee --- main/models.py | 4 ++-- packages/models.py | 20 ++++++++------------ 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/main/models.py b/main/models.py index 46fd3a63..da9d8b6e 100644 --- a/main/models.py +++ b/main/models.py @@ -277,10 +277,10 @@ class Package(models.Model): # TODO: we can use list comprehension and an 'in' query to make this # more effective for dep in self.depends.all(): - pkg = dep.get_best_satisfier(self) + pkg = dep.get_best_satisfier() providers = None if not pkg: - providers = dep.get_providers(self) + providers = dep.get_providers() deps.append({'dep': dep, 'pkg': pkg, 'providers': providers}) # sort the list; deptype sorting makes this tricker than expected sort_order = {'D': 0, 'O': 1, 'M': 2, 'C': 3} diff --git a/packages/models.py b/packages/models.py index ff677883..7bcdc000 100644 --- a/packages/models.py +++ b/packages/models.py @@ -357,16 +357,14 @@ class RelatedToBase(models.Model): name = models.CharField(max_length=255, db_index=True) version = models.CharField(max_length=255, default='') - def get_best_satisfier(self, main_pkg=None): + def get_best_satisfier(self): '''Find a satisfier for this related package that best matches the given criteria. It will not search provisions, but will find packages named and matching repo characteristics if possible.''' - if main_pkg is None: - main_pkg = self.pkg pkgs = Package.objects.normal().filter(pkgname=self.name) - if not main_pkg.arch.agnostic: + if not self.pkg.arch.agnostic: # make sure we match architectures if possible - arches = main_pkg.applicable_arches() + arches = self.pkg.applicable_arches() pkgs = pkgs.filter(arch__in=arches) # if we have a comparison operation, make sure the packages we grab # actually satisfy the requirements @@ -386,27 +384,25 @@ class RelatedToBase(models.Model): pkg = pkgs[0] # prevents yet more DB queries, these lists should be short; # after each grab the best available in case we remove all entries - pkgs = [p for p in pkgs if p.repo.staging == main_pkg.repo.staging] + pkgs = [p for p in pkgs if p.repo.staging == self.pkg.repo.staging] if len(pkgs) > 0: pkg = pkgs[0] - pkgs = [p for p in pkgs if p.repo.testing == main_pkg.repo.testing] + pkgs = [p for p in pkgs if p.repo.testing == self.pkg.repo.testing] if len(pkgs) > 0: pkg = pkgs[0] return pkg - def get_providers(self, main_pkg=None): + def get_providers(self): '''Return providers of this related package. Does *not* include exact matches as it checks the Provision names only, use get_best_satisfier() instead for exact matches.''' - if main_pkg is None: - main_pkg = self.pkg pkgs = Package.objects.normal().filter( provides__name=self.name).order_by().distinct() - if not main_pkg.arch.agnostic: + if not self.pkg.arch.agnostic: # make sure we match architectures if possible - arches = main_pkg.applicable_arches() + arches = self.pkg.applicable_arches() pkgs = pkgs.filter(arch__in=arches) # If we have a comparison operation, make sure the packages we grab -- cgit v1.1-4-g5e80 From 8524a6057c4b99a620850494a22eb3d1f56bee68 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 20 Jan 2013 14:15:53 -0600 Subject: Change caching strategy on package.applicable_arches Rather than use the Django cache for this (aka memcached), just do it on a per-object instantiation basis. This means no external services calls except the first time to the database, which should be quite a bit faster. Signed-off-by: Dan McGee --- main/models.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/main/models.py b/main/models.py index da9d8b6e..ec5c2bd6 100644 --- a/main/models.py +++ b/main/models.py @@ -177,12 +177,15 @@ class Package(models.Model): def maintainers(self, maintainers): self._maintainers = maintainers - @cache_function(1800) + _applicable_arches = None + def applicable_arches(self): '''The list of (this arch) + (available agnostic arches).''' - arches = set(Arch.objects.filter(agnostic=True)) - arches.add(self.arch) - return list(arches) + if self._applicable_arches is None: + arches = set(Arch.objects.filter(agnostic=True)) + arches.add(self.arch) + self._applicable_arches = list(arches) + return self._applicable_arches @cache_function(119) def get_requiredby(self): -- cgit v1.1-4-g5e80 From 5566d43a7734f6bb2f48d5d511351da12ddc5cc1 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 9 Feb 2013 16:43:40 -0600 Subject: Use 'update_fields' model.save() kwarg This was added in Django 1.5 and allows saving only a subset of a model's fields. It makes sense in a few cases to utilize it. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorresolv.py | 2 +- packages/views/signoff.py | 2 +- releng/management/commands/syncisos.py | 2 +- todolists/views.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mirrors/management/commands/mirrorresolv.py b/mirrors/management/commands/mirrorresolv.py index 0370f8ed..a6c2523e 100644 --- a/mirrors/management/commands/mirrorresolv.py +++ b/mirrors/management/commands/mirrorresolv.py @@ -53,7 +53,7 @@ def resolve_mirrors(): newvals = (mirrorurl.has_ipv4, mirrorurl.has_ipv6) if newvals != oldvals: logger.debug("values changed for %s", mirrorurl) - mirrorurl.save(force_update=True) + mirrorurl.save(update_fields=('has_ipv4', 'has_ipv6')) except socket.error, e: logger.warn("error resolving %s: %s", mirrorurl.hostname, e) diff --git a/packages/views/signoff.py b/packages/views/signoff.py index 17f3095c..c37aa0fc 100644 --- a/packages/views/signoff.py +++ b/packages/views/signoff.py @@ -45,7 +45,7 @@ def signoff_package(request, name, repo, arch, revoke=False): except Signoff.DoesNotExist: raise Http404 signoff.revoked = now() - signoff.save() + signoff.save(update_fields=('revoked',)) created = False else: # ensure we should even be accepting signoffs diff --git a/releng/management/commands/syncisos.py b/releng/management/commands/syncisos.py index 223c771b..c9f61964 100644 --- a/releng/management/commands/syncisos.py +++ b/releng/management/commands/syncisos.py @@ -53,7 +53,7 @@ class Command(BaseCommand): if not existing.active: existing.active = True existing.removed = None - existing.save() + existing.save(update_fields=('active', 'removed')) # and then mark all other names as no longer active Iso.objects.filter(active=True).exclude(name__in=active_isos).update( active=False, removed=now()) diff --git a/todolists/views.py b/todolists/views.py index f333728a..9935987b 100644 --- a/todolists/views.py +++ b/todolists/views.py @@ -47,7 +47,7 @@ def flag(request, slug, pkg_id): else: tlpkg.status = TodolistPackage.INCOMPLETE tlpkg.user = request.user - tlpkg.save() + tlpkg.save(update_fields=('status', 'user', 'last_modified')) if request.is_ajax(): data = { 'status': tlpkg.get_status_display(), -- cgit v1.1-4-g5e80 From a25cc4ff459ab0070ce48730f6c6fdc227860f08 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 26 Feb 2013 18:44:37 -0600 Subject: Bump Django requirement to 1.5 Signed-off-by: Dan McGee --- requirements.txt | 2 +- requirements_prod.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index b8a06215..24f83d57 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin -Django==1.4.3 +Django==1.5 Markdown==2.2.1 South==0.7.6 bencode==1.0 diff --git a/requirements_prod.txt b/requirements_prod.txt index 705c56bb..0913d32a 100644 --- a/requirements_prod.txt +++ b/requirements_prod.txt @@ -1,5 +1,5 @@ -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin -Django==1.4.3 +Django==1.5 Markdown==2.2.1 South==0.7.6 bencode==1.0 -- cgit v1.1-4-g5e80