From a60801bb7dbc18080e7f6106bcf9c707d2801c9d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 11 Dec 2011 19:52:27 -0600 Subject: PyLint suggested cleanups Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 2 +- devel/tests.py | 2 +- devel/views.py | 18 +++++++++--------- 3 files changed, 11 insertions(+), 11 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index cc536b15..7e058afc 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -24,7 +24,6 @@ from datetime import datetime from optparse import make_option from django.core.management.base import BaseCommand, CommandError -from django.contrib.auth.models import User from django.db import connections, router, transaction from django.db.utils import IntegrityError @@ -91,6 +90,7 @@ class Pkg(object): setattr(self, k, None) for k in self.collections: setattr(self, k, ()) + self.builddate = None self.files = None self.has_files = False diff --git a/devel/tests.py b/devel/tests.py index 36691179..01eed0fc 100644 --- a/devel/tests.py +++ b/devel/tests.py @@ -99,7 +99,7 @@ class FindUserTest(TestCase): def test_cache(self): # simply look two of them up, but then do it repeatedly - for i in range(50): + for _ in range(5): self.assertEqual(self.user1, self.finder.find("XXX YYY ")) self.assertEqual(self.user3, diff --git a/devel/views.py b/devel/views.py index 79272c98..2e62200e 100644 --- a/devel/views.py +++ b/devel/views.py @@ -145,7 +145,7 @@ def change_profile(request): {'form': form, 'profile_form': profile_form}) @login_required -def report(request, report, username=None): +def report(request, report_name, username=None): title = 'Developer Report' packages = Package.objects.normal() names = attrs = user = None @@ -159,17 +159,17 @@ def report(request, report, username=None): maints = User.objects.filter(id__in=PackageRelation.objects.filter( type=PackageRelation.MAINTAINER).values('user')) - if report == 'old': + if report_name == 'old': title = 'Packages last built more than two years ago' cutoff = datetime.utcnow() - timedelta(days=365 * 2) packages = packages.filter( build_date__lt=cutoff).order_by('build_date') - elif report == 'long-out-of-date': + elif report_name == 'long-out-of-date': title = 'Packages marked out-of-date more than 90 days ago' cutoff = datetime.utcnow() - timedelta(days=90) packages = packages.filter( flag_date__lt=cutoff).order_by('flag_date') - elif report == 'big': + elif report_name == 'big': title = 'Packages with compressed size > 50 MiB' cutoff = 50 * 1024 * 1024 packages = packages.filter( @@ -182,7 +182,7 @@ def report(request, report, username=None): package.compressed_size) package.installed_size_pretty = filesizeformat( package.installed_size) - elif report == 'badcompression': + elif report_name == 'badcompression': title = 'Packages that have little need for compression' cutoff = 0.90 * F('installed_size') packages = packages.filter(compressed_size__gt=0, installed_size__gt=0, @@ -199,7 +199,7 @@ def report(request, report, username=None): ratio = package.compressed_size / float(package.installed_size) package.ratio = '%.2f' % ratio package.compress_type = package.filename.split('.')[-1] - elif report == 'uncompressed-man': + elif report_name == 'uncompressed-man': title = 'Packages with uncompressed manpages' # checking for all '.0'...'.9' + '.n' extensions bad_files = PackageFile.objects.filter(directory__contains='/man/', @@ -209,7 +209,7 @@ def report(request, report, username=None): bad_files = bad_files.filter(pkg__in=pkg_ids) bad_files = bad_files.values_list('pkg_id', flat=True).distinct() packages = packages.filter(id__in=set(bad_files)) - elif report == 'uncompressed-info': + elif report_name == 'uncompressed-info': title = 'Packages with uncompressed infopages' # we don't worry about looking for '*.info-1', etc., given that an # uncompressed root page probably exists in the package anyway @@ -220,7 +220,7 @@ def report(request, report, username=None): bad_files = bad_files.filter(pkg__in=pkg_ids) bad_files = bad_files.values_list('pkg_id', flat=True).distinct() packages = packages.filter(id__in=set(bad_files)) - elif report == 'unneeded-orphans': + elif report_name == 'unneeded-orphans': title = 'Orphan packages required by no other packages' owned = PackageRelation.objects.all().values('pkgbase') required = PackageDepend.objects.all().values('depname') @@ -273,7 +273,7 @@ class NewUserForm(forms.ModelForm): def save(self, commit=True): profile = super(NewUserForm, self).save(False) pwletters = ascii_letters + digits - password = ''.join([random.choice(pwletters) for i in xrange(8)]) + password = ''.join([random.choice(pwletters) for _ in xrange(8)]) user = User.objects.create_user(username=self.cleaned_data['username'], email=self.cleaned_data['private_email'], password=password) user.first_name = self.cleaned_data['first_name'] -- cgit v1.2.3-2-g168b From 56af1e4f50587333da4e8c38800be8f720af98b7 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 12 Dec 2011 09:58:53 -0600 Subject: reporead: more efficient deletion of files Rather than delegating to Django and batch deletion by ID, force issuing of a single delete query to clear out all existing file objects when necessary. This should speed up the deletion and update of packages with a lot of files by a non-trivial amount. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 7e058afc..aa04a31f 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -239,6 +239,12 @@ pkg_same_version = lambda pkg, dbpkg: pkg.ver == dbpkg.pkgver \ and pkg.rel == dbpkg.pkgrel and pkg.epoch == dbpkg.epoch +def delete_pkg_files(dbpkg): + database = router.db_for_write(Package, instance=dbpkg) + cursor = connections[database].cursor() + cursor.execute('DELETE FROM package_files WHERE pkg_id = %s', [dbpkg.id]) + + def populate_files(dbpkg, repopkg, force=False): if not force: if not pkg_same_version(repopkg, dbpkg): @@ -253,7 +259,7 @@ def populate_files(dbpkg, repopkg, force=False): # only delete files if we are reading a DB that contains them if repopkg.has_files: - dbpkg.packagefile_set.all().delete() + delete_pkg_files(dbpkg) logger.info("adding %d files for package %s", len(repopkg.files), dbpkg.pkgname) for f in repopkg.files: @@ -262,6 +268,7 @@ def populate_files(dbpkg, repopkg, force=False): filename = None # this is basically like calling dbpkg.packagefile_set.create(), # but much faster as we can skip a lot of the repeated code paths + # TODO use Django 1.4 bulk_create pkgfile = PackageFile(pkg=dbpkg, is_directory=(filename is None), directory=dirname + '/', @@ -361,6 +368,7 @@ def db_update(archname, reponame, pkgs, force=False): with transaction.commit_on_success(): # no race condition here as long as simultaneous threads both # issue deletes; second delete will be a no-op + delete_pkg_files(dbpkg) dbpkg.delete() # packages in both database and in syncdb (update in database) -- cgit v1.2.3-2-g168b From ab97f5cdea4d54bbb8dff34b0333f5b965052e91 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 22 Dec 2011 10:22:25 -0600 Subject: Fix mismatched URL keyword arg Signed-off-by: Dan McGee --- devel/urls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'devel') diff --git a/devel/urls.py b/devel/urls.py index bc3bcace..07cb321b 100644 --- a/devel/urls.py +++ b/devel/urls.py @@ -7,8 +7,8 @@ urlpatterns = patterns('devel.views', (r'^$', 'index', {}, 'devel-index'), (r'^newuser/$', 'new_user_form'), (r'^profile/$', 'change_profile'), - (r'^reports/(?P.*)/(?P.*)/$', 'report'), - (r'^reports/(?P.*)/$', 'report'), + (r'^reports/(?P.*)/(?P.*)/$', 'report'), + (r'^reports/(?P.*)/$', 'report'), ) # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From 6b16b9487a95118a6109a2c5119d430dc1192e80 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 5 Jan 2012 13:03:00 -0600 Subject: Adjust page and content caching lengths and decorators Remove never_cache from many places now that we don't actually need it since we aren't caching by default. Adjust our cache_function decorator times be shorter values, and also randomize them a bit to make cache invalidations not all line up. Signed-off-by: Dan McGee --- devel/views.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'devel') diff --git a/devel/views.py b/devel/views.py index 2e62200e..35918d66 100644 --- a/devel/views.py +++ b/devel/views.py @@ -13,6 +13,7 @@ from django.template import loader, Context from django.template.defaultfilters import filesizeformat from django.views.decorators.cache import never_cache from django.views.generic.simple import direct_to_template +from django.utils.http import http_date from main.models import Package, PackageDepend, PackageFile, TodolistPkg from main.models import Arch, Repo @@ -27,9 +28,9 @@ import operator import pytz import random from string import ascii_letters, digits +import time @login_required -@never_cache def index(request): '''the developer dashboard''' if(request.user.is_authenticated()): @@ -80,7 +81,6 @@ def index(request): return direct_to_template(request, 'devel/index.html', page_dict) @login_required -@never_cache def clock(request): devs = User.objects.filter(is_active=True).order_by( 'first_name', 'last_name').select_related('userprofile') @@ -98,7 +98,14 @@ def clock(request): 'utc_now': utc_now, } - return direct_to_template(request, 'devel/clock.html', page_dict) + response = direct_to_template(request, 'devel/clock.html', page_dict) + if not response.has_header('Expires'): + # why this works only with the non-UTC date I have no idea... + expire_time = now.replace(minute=utc_now.minute + 1, + second=0, microsecond=0) + expire_time = time.mktime(expire_time.timetuple()) + response['Expires'] = http_date(expire_time) + return response class ProfileForm(forms.Form): email = forms.EmailField(label='Private email (not shown publicly):', @@ -342,7 +349,6 @@ def new_user_form(request): return direct_to_template(request, 'general_form.html', context) @user_passes_test(lambda u: u.is_superuser) -@never_cache def admin_log(request, username=None): user = None if username: -- cgit v1.2.3-2-g168b