From 133d16f91f3d296e188d910f609128d854e65823 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 29 Mar 2013 15:51:50 -0500 Subject: Add IP family lookup to CheckLocation model Signed-off-by: Dan McGee --- mirrors/models.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mirrors/models.py b/mirrors/models.py index e41f6b22..b0da5616 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -136,6 +136,15 @@ class CheckLocation(models.Model): families = [x[0] for x in info] return families[0] + @property + def ip_version(self): + '''Returns integer '4' or '6'.''' + if self.family == socket.AF_INET6: + return 6 + if self.family == socket.AF_INET: + return 4 + return None + class MirrorLog(models.Model): url = models.ForeignKey(MirrorUrl, related_name="logs") -- cgit v1.1-4-g5e80 From 25aa164823a88f13761c750057833b6724808675 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 19 Mar 2013 00:10:52 -0500 Subject: Remove old-style build date parsing This was added in 2010 in commit e95c4563e32 as a short-term fix. The short-term is up. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index ccac55f2..a0e77dc7 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -116,13 +116,9 @@ class RepoPackage(object): builddate = datetime.utcfromtimestamp(int(v[0])) self.builddate = builddate.replace(tzinfo=utc) except ValueError: - try: - self.builddate = datetime.strptime(v[0], - '%a %b %d %H:%M:%S %Y') - except ValueError: - logger.warning( - 'Package %s had unparsable build date %s', - self.name, v[0]) + logger.warning( + 'Package %s had unparsable build date %s', + self.name, v[0]) elif k == 'files': self.files = tuple(v) self.has_files = True -- cgit v1.1-4-g5e80 From 25b18cb3c45411967772022473865076735e7105 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 23 Mar 2013 11:50:06 -0500 Subject: Update some package requirements Signed-off-by: Dan McGee --- requirements.txt | 6 +++--- requirements_prod.txt | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 24f83d57..ab81387f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin -Django==1.5 -Markdown==2.2.1 +Django==1.5.1 +Markdown==2.3.1 South==0.7.6 bencode==1.0 django-countries==1.5 jsmin==2.0.2-1 pgpdump==1.4 -pytz>=2012j +pytz>=2013b diff --git a/requirements_prod.txt b/requirements_prod.txt index 0913d32a..6a0a78f2 100644 --- a/requirements_prod.txt +++ b/requirements_prod.txt @@ -1,6 +1,6 @@ -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin -Django==1.5 -Markdown==2.2.1 +Django==1.5.1 +Markdown==2.3.1 South==0.7.6 bencode==1.0 django-countries==1.5 @@ -9,4 +9,4 @@ pgpdump==1.4 psycopg2==2.4.6 pyinotify==0.9.4 python-memcached==1.48 -pytz>=2012j +pytz>=2013b -- cgit v1.1-4-g5e80 From 90e969a160e1ec028ded1ca9b33975ec50fed154 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 8 Apr 2013 08:33:34 -0500 Subject: Bump psycopg2 requirements.txt version Signed-off-by: Dan McGee --- requirements_prod.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_prod.txt b/requirements_prod.txt index 6a0a78f2..08bfc27c 100644 --- a/requirements_prod.txt +++ b/requirements_prod.txt @@ -6,7 +6,7 @@ bencode==1.0 django-countries==1.5 jsmin==2.0.2-1 pgpdump==1.4 -psycopg2==2.4.6 +psycopg2==2.5 pyinotify==0.9.4 python-memcached==1.48 pytz>=2013b -- cgit v1.1-4-g5e80 From 06e1e857abfdf7f95661d337ce3c315bd51fb837 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 10 Apr 2013 21:00:17 -0500 Subject: Allow mirror rsync IPs to be IPv4/IPv6 addresses or networks This gives us a bunch more flexibility on this field, and now supports all the options that the rsync config file supports. Signed-off-by: Dan McGee --- mirrors/admin.py | 14 ---- mirrors/fields.py | 49 +++++++++++++ .../0025_auto__chg_field_mirrorrsync_ip.py | 85 ++++++++++++++++++++++ mirrors/models.py | 3 +- requirements.txt | 1 + requirements_prod.txt | 1 + 6 files changed, 138 insertions(+), 15 deletions(-) create mode 100644 mirrors/fields.py create mode 100644 mirrors/migrations/0025_auto__chg_field_mirrorrsync_ip.py diff --git a/mirrors/admin.py b/mirrors/admin.py index d6ea3950..9c88207d 100644 --- a/mirrors/admin.py +++ b/mirrors/admin.py @@ -1,4 +1,3 @@ -import re from urlparse import urlparse, urlunsplit from django import forms @@ -36,22 +35,9 @@ class MirrorUrlInlineAdmin(admin.TabularInline): extra = 3 -# ripped off from django.forms.fields, adding netmask ability -IPV4NM_RE = re.compile(r'^(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}(/(\d|[1-2]\d|3[0-2])){0,1}$') - -class IPAddressNetmaskField(forms.fields.RegexField): - default_error_messages = { - 'invalid': u'Enter a valid IPv4 address, possibly including netmask.', - } - - def __init__(self, *args, **kwargs): - super(IPAddressNetmaskField, self).__init__(IPV4NM_RE, *args, **kwargs) - - class MirrorRsyncForm(forms.ModelForm): class Meta: model = MirrorRsync - ip = IPAddressNetmaskField(label='IP') class MirrorRsyncInlineAdmin(admin.TabularInline): diff --git a/mirrors/fields.py b/mirrors/fields.py new file mode 100644 index 00000000..206c9d7d --- /dev/null +++ b/mirrors/fields.py @@ -0,0 +1,49 @@ +from IPy import IP + +from django import forms +from django.core import validators +from django.core.exceptions import ValidationError +from django.db import models +from south.modelsinspector import add_introspection_rules + + +class IPNetworkFormField(forms.Field): + def to_python(self, value): + if value in validators.EMPTY_VALUES: + return None + try: + value = IP(value) + except ValueError as e: + raise ValidationError(str(e)) + return value + + +class IPNetworkField(models.Field): + __metaclass__ = models.SubfieldBase + description = "IPv4 or IPv6 address or subnet" + + def __init__(self, *args, **kwargs): + kwargs['max_length'] = 44 + super(IPNetworkField, self).__init__(*args, **kwargs) + + def get_internal_type(self): + return "IPAddressField" + + def to_python(self, value): + if not value: + return None + return IP(value) + + def get_prep_value(self, value): + value = self.to_python(value) + if not value: + return None + return str(value) + + def formfield(self, **kwargs): + defaults = {'form_class': IPNetworkFormField} + defaults.update(kwargs) + return super(IPNetworkField, self).formfield(**defaults) + + +add_introspection_rules([], ["^mirrors\.fields\.IPNetworkField"]) diff --git a/mirrors/migrations/0025_auto__chg_field_mirrorrsync_ip.py b/mirrors/migrations/0025_auto__chg_field_mirrorrsync_ip.py new file mode 100644 index 00000000..b359b637 --- /dev/null +++ b/mirrors/migrations/0025_auto__chg_field_mirrorrsync_ip.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + if db.backend_name == 'postgres': + # For PostgreSQL, because it uses the 'inet' type and not a varchar + # column, we need to add an explict 'USING' cast to the SQL + # statement. We then execute the alter_column as well to ensure any + # of the other side-effects happen. + db.execute('ALTER TABLE "mirrors_mirrorrsync" ALTER COLUMN "ip" TYPE inet USING "ip"::inet') + db.alter_column(u'mirrors_mirrorrsync', 'ip', self.gf('mirrors.fields.IPNetworkField')(max_length=44)) + + def backwards(self, orm): + db.alter_column(u'mirrors_mirrorrsync', 'ip', self.gf('django.db.models.fields.CharField')(max_length=44)) + + models = { + u'mirrors.checklocation': { + 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'}, + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'}) + }, + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'location': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'null': 'True', 'to': u"orm['mirrors.CheckLocation']"}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('mirrors.fields.IPNetworkField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index b0da5616..791b0078 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -6,6 +6,7 @@ from django.db import models from django.db.models.signals import pre_save from django_countries import CountryField +from .fields import IPNetworkField from main.utils import set_created_field @@ -105,7 +106,7 @@ class MirrorUrl(models.Model): class MirrorRsync(models.Model): # max length is 40 chars for full-form IPv6 addr + subnet - ip = models.CharField("IP", max_length=44) + ip = IPNetworkField("IP") mirror = models.ForeignKey(Mirror, related_name="rsync_ips") created = models.DateTimeField(editable=False) diff --git a/requirements.txt b/requirements.txt index ab81387f..84450c76 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin Django==1.5.1 +IPy==0.81 Markdown==2.3.1 South==0.7.6 bencode==1.0 diff --git a/requirements_prod.txt b/requirements_prod.txt index 08bfc27c..cde5f513 100644 --- a/requirements_prod.txt +++ b/requirements_prod.txt @@ -1,5 +1,6 @@ -e git+git://github.com/fredj/cssmin.git@master#egg=cssmin Django==1.5.1 +IPy==0.81 Markdown==2.3.1 South==0.7.6 bencode==1.0 -- cgit v1.1-4-g5e80 From 2c24ee9100a9e60fec16055d6496caeda3a1d8e2 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 13 Apr 2013 11:33:17 -0500 Subject: Calculate average URL delay in the database Rather than doing this in the Python code and needing 12,000+ rows returned from the database, we can do it in the database and get fewer than 300 rows back. If I recall correctly, the reason this was not done originally was due to our usage of MySQL and some really bad date math/overflow stuff it did when the interval between last_sync and check_time were greater than about a week. Luckily, we have switched to using a more sane database. Signed-off-by: Dan McGee --- mirrors/utils.py | 49 ++++++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/mirrors/utils.py b/mirrors/utils.py index 3ab176b3..2721e20e 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -1,5 +1,6 @@ from datetime import timedelta +from django.db import connection from django.db.models import Avg, Count, Max, Min, StdDev from django.utils.timezone import now from django_countries.fields import Country @@ -10,13 +11,12 @@ from .models import MirrorLog, MirrorProtocol, MirrorUrl DEFAULT_CUTOFF = timedelta(hours=24) -def annotate_url(url, delays): +def annotate_url(url, delay): '''Given a MirrorURL object, add a few more attributes to it regarding status, including completion_pct, delay, and score.''' url.completion_pct = float(url.success_count) / url.check_count - if url.id in delays: - url_delays = delays[url.id] - url.delay = sum(url_delays, timedelta()) / len(url_delays) + if delay is not None: + url.delay = delay hours = url.delay.days * 24.0 + url.delay.seconds / 3600.0 if url.completion_pct > 0: @@ -30,6 +30,30 @@ def annotate_url(url, delays): url.score = None +def url_delays(cutoff_time, mirror_id=None): + cursor = connection.cursor() + if mirror_id is None: + sql= """ +SELECT url_id, AVG(check_time - last_sync) +FROM mirrors_mirrorlog +WHERE is_success = %s AND check_time >= %s AND last_sync IS NOT NULL +GROUP BY url_id +""" + cursor.execute(sql, [True, cutoff_time]) + else: + sql = """ +SELECT l.url_id, avg(check_time - last_sync) +FROM mirrors_mirrorlog l +JOIN mirrors_mirrorurl u ON u.id = l.url_id +WHERE is_success = %s AND check_time >= %s AND last_sync IS NOT NULL +AND mirror_id = %s +GROUP BY url_id +""" + cursor.execute(sql, [True, cutoff_time, mirror_id]) + + return {url_id: delay for url_id, delay in cursor.fetchall()} + + @cache_function(123) def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): cutoff_time = now() - cutoff @@ -55,20 +79,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( id__in=valid_urls).order_by('mirror__id', 'url') - - # The Django ORM makes it really hard to get actual average delay in the - # above query, so run a seperate query for it and we will process the - # results here. - times = MirrorLog.objects.values_list( - 'url_id', 'check_time', 'last_sync').filter( - is_success=True, last_sync__isnull=False, - check_time__gte=cutoff_time) - if mirror_ids: - times = times.filter(url__mirror_id__in=mirror_ids) - delays = {} - for url_id, check_time, last_sync in times: - delay = check_time - last_sync - delays.setdefault(url_id, []).append(delay) + delays = url_delays(cutoff_time) if urls: url_data = dict((item['id'], item) for item in url_data) @@ -97,7 +108,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): # fake the standard deviation for local testing setups if vendor == 'sqlite': setattr(url, 'duration_stddev', 0.0) - annotate_url(url, delays) + annotate_url(url, delays.get(url.id, None)) return { 'cutoff': cutoff, -- cgit v1.1-4-g5e80 From c588d1c85f86f5ee10a96bec679111c8675b703c Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 13 Apr 2013 11:38:11 -0500 Subject: Support only a single mirror ID in error/status retrieval This simplifies things and makes injecting this single mirror ID into custom SQL a whole lot easier. Signed-off-by: Dan McGee --- mirrors/utils.py | 18 +++++++++--------- mirrors/views.py | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mirrors/utils.py b/mirrors/utils.py index 2721e20e..d18dc22f 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -55,15 +55,15 @@ GROUP BY url_id @cache_function(123) -def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): +def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): cutoff_time = now() - cutoff valid_urls = MirrorUrl.objects.filter( mirror__active=True, mirror__public=True, logs__check_time__gte=cutoff_time).distinct() - if mirror_ids: - valid_urls = valid_urls.filter(mirror_id__in=mirror_ids) + if mirror_id: + valid_urls = valid_urls.filter(mirror_id=mirror_id) url_data = MirrorUrl.objects.values('id', 'mirror_id').filter( id__in=valid_urls, logs__check_time__gte=cutoff_time).annotate( @@ -79,7 +79,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( id__in=valid_urls).order_by('mirror__id', 'url') - delays = url_delays(cutoff_time) + delays = url_delays(cutoff_time, mirror_id) if urls: url_data = dict((item['id'], item) for item in url_data) @@ -90,8 +90,8 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): last_check = max([u.last_check for u in urls]) num_checks = max([u.check_count for u in urls]) check_info = MirrorLog.objects.filter(check_time__gte=cutoff_time) - if mirror_ids: - check_info = check_info.filter(url__mirror_id__in=mirror_ids) + if mirror_id: + check_info = check_info.filter(url__mirror_id=mirror_id) check_info = check_info.aggregate( mn=Min('check_time'), mx=Max('check_time')) if num_checks > 1: @@ -120,7 +120,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): @cache_function(117) -def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None): +def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None): cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( is_success=False, check_time__gte=cutoff_time, @@ -130,8 +130,8 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None): error_count=Count('error'), last_occurred=Max('check_time') ).order_by('-last_occurred', '-error_count') - if mirror_ids: - urls = urls.filter(mirror_id__in=mirror_ids) + if mirror_id: + urls = urls.filter(mirror_id=mirror_id) errors = list(errors) for err in errors: diff --git a/mirrors/views.py b/mirrors/views.py index 56397633..07e28d40 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -175,7 +175,7 @@ def mirror_details(request, name): (not mirror.public or not mirror.active): raise Http404 - status_info = get_mirror_statuses(mirror_ids=[mirror.id]) + status_info = get_mirror_statuses(mirror_id=mirror.id) checked_urls = {url for url in status_info['urls'] \ if url.mirror_id == mirror.id} all_urls = set(mirror.urls.select_related('protocol')) @@ -193,7 +193,7 @@ def mirror_details(request, name): def mirror_details_json(request, name): mirror = get_object_or_404(Mirror, name=name) - status_info = get_mirror_statuses(mirror_ids=[mirror.id]) + status_info = get_mirror_statuses(mirror_id=mirror.id) data = status_info.copy() data['version'] = 3 to_json = json.dumps(data, ensure_ascii=False, -- cgit v1.1-4-g5e80 From f2a6316be0b025a9ee22f22d34df1c00f60a8bdf Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 13 Apr 2013 11:56:26 -0500 Subject: Add additional pg_trgm indexes for quicker searches This allows our normal keyword-based search to be index-optimized rather than always doing full table scans. It requires the pg_trgm extension which is shipped out of the box with any sane install of PostgreSQL. Signed-off-by: Dan McGee --- packages/sql/search_indexes.postgresql_psycopg2.sql | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 packages/sql/search_indexes.postgresql_psycopg2.sql diff --git a/packages/sql/search_indexes.postgresql_psycopg2.sql b/packages/sql/search_indexes.postgresql_psycopg2.sql new file mode 100644 index 00000000..a7eaf998 --- /dev/null +++ b/packages/sql/search_indexes.postgresql_psycopg2.sql @@ -0,0 +1,3 @@ +CREATE EXTENSION IF NOT EXISTS pg_trgm; +CREATE INDEX packages_pkgname_trgm_gist ON packages USING gist (UPPER(pkgname) gist_trgm_ops); +CREATE INDEX packages_pkgdesc_trgm_gist ON packages USING gist (UPPER(pkgdesc) gist_trgm_ops); -- cgit v1.1-4-g5e80 From 7fc8da7d959556b1204b7864959e73e7f5f5ec59 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 13 Apr 2013 13:05:02 -0500 Subject: Show replacments for package if it has been removed This covers the case where we can't find the package in any other repositories, but it was removed recently enough that we have a found package update object. Signed-off-by: Dan McGee --- packages/models.py | 9 +++++++++ packages/views/display.py | 2 ++ 2 files changed, 11 insertions(+) diff --git a/packages/models.py b/packages/models.py index 7bcdc000..92566a56 100644 --- a/packages/models.py +++ b/packages/models.py @@ -321,6 +321,15 @@ class Update(models.Model): return Package.objects.normal().filter( pkgname=self.pkgname, arch=self.arch) + def replacements(self): + pkgs = Package.objects.normal().filter( + replaces__name=self.pkgname) + if not self.arch.agnostic: + # make sure we match architectures if possible + arches = self.pkg.applicable_arches() + pkgs = pkgs.filter(arch__in=arches) + return pkgs + def __unicode__(self): return u'%s of %s on %s' % (self.get_action_flag_display(), self.pkgname, self.created) diff --git a/packages/views/display.py b/packages/views/display.py index fcf8fdea..50783835 100644 --- a/packages/views/display.py +++ b/packages/views/display.py @@ -55,6 +55,8 @@ def recently_removed_package(request, name, repo, arch, cutoff=CUTOFF): try: update = match.latest() elsewhere = update.elsewhere() + if len(elsewhere) == 0: + elsewhere = update.replacements() if len(elsewhere) == 1: return redirect(elsewhere[0]) context = { -- cgit v1.1-4-g5e80 From 213aa3a2fab6f3a56be348a067c132f568efbaff Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 14 Apr 2013 13:09:41 -0500 Subject: Reduce mirror status query madness Move completely to custom SQL for this logic. The Django ORM just doesn't play nice with the kind of query we are looking to do, so it is easier to do using raw SQL. The biggest pain factor here is in supporting sqlite as it doesn't have nearly the capabilities in handling datetime types directly in the database, as well as having some different type conversion necessities. Signed-off-by: Dan McGee --- mirrors/utils.py | 146 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 91 insertions(+), 55 deletions(-) diff --git a/mirrors/utils.py b/mirrors/utils.py index d18dc22f..eb1211f1 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -2,6 +2,7 @@ from datetime import timedelta from django.db import connection from django.db.models import Avg, Count, Max, Min, StdDev +from django.utils.dateparse import parse_datetime from django.utils.timezone import now from django_countries.fields import Country @@ -11,47 +12,103 @@ from .models import MirrorLog, MirrorProtocol, MirrorUrl DEFAULT_CUTOFF = timedelta(hours=24) -def annotate_url(url, delay): - '''Given a MirrorURL object, add a few more attributes to it regarding - status, including completion_pct, delay, and score.''' - url.completion_pct = float(url.success_count) / url.check_count - if delay is not None: - url.delay = delay - hours = url.delay.days * 24.0 + url.delay.seconds / 3600.0 - if url.completion_pct > 0: - divisor = url.completion_pct - else: - # arbitrary small value - divisor = 0.005 - url.score = (hours + url.duration_avg + url.duration_stddev) / divisor - else: - url.delay = None - url.score = None +def dictfetchall(cursor): + "Returns all rows from a cursor as a dict." + desc = cursor.description + return [ + dict(zip([col[0] for col in desc], row)) + for row in cursor.fetchall() + ] -def url_delays(cutoff_time, mirror_id=None): - cursor = connection.cursor() - if mirror_id is None: - sql= """ -SELECT url_id, AVG(check_time - last_sync) -FROM mirrors_mirrorlog -WHERE is_success = %s AND check_time >= %s AND last_sync IS NOT NULL -GROUP BY url_id +def status_data(cutoff_time, mirror_id=None): + if mirror_id is not None: + params = [cutoff_time, mirror_id] + mirror_where = 'AND u.mirror_id = %s' + else: + params = [cutoff_time] + mirror_where = '' + + vendor = database_vendor(MirrorUrl) + if vendor == 'sqlite': + sql = """ +SELECT l.url_id, u.mirror_id, + COUNT(l.id) AS check_count, + COUNT(l.duration) AS success_count, + MAX(l.last_sync) AS last_sync, + MAX(l.check_time) AS last_check, + AVG(l.duration) AS duration_avg, + 0.0 AS duration_stddev, + AVG(STRFTIME('%%s', check_time) - STRFTIME('%%s', last_sync)) AS delay +FROM mirrors_mirrorlog l +JOIN mirrors_mirrorurl u ON u.id = l.url_id +WHERE l.check_time >= %s +""" + mirror_where + """ +GROUP BY l.url_id, u.mirror_id """ - cursor.execute(sql, [True, cutoff_time]) else: sql = """ -SELECT l.url_id, avg(check_time - last_sync) +SELECT l.url_id, u.mirror_id, + COUNT(l.id) AS check_count, + COUNT(l.duration) AS success_count, + MAX(l.last_sync) AS last_sync, + MAX(l.check_time) AS last_check, + AVG(l.duration) AS duration_avg, + STDDEV(l.duration) AS duration_stddev, + AVG(check_time - last_sync) AS delay FROM mirrors_mirrorlog l JOIN mirrors_mirrorurl u ON u.id = l.url_id -WHERE is_success = %s AND check_time >= %s AND last_sync IS NOT NULL -AND mirror_id = %s -GROUP BY url_id +WHERE l.check_time >= %s +""" + mirror_where + """ +GROUP BY l.url_id, u.mirror_id """ - cursor.execute(sql, [True, cutoff_time, mirror_id]) - return {url_id: delay for url_id, delay in cursor.fetchall()} + cursor = connection.cursor() + cursor.execute(sql, params) + url_data = dictfetchall(cursor) + + # sqlite loves to return less than ideal types + if vendor == 'sqlite': + for item in url_data: + item['delay'] = timedelta(seconds=item['delay']) + item['last_sync'] = parse_datetime(item['last_sync']) + item['last_check'] = parse_datetime(item['last_check']) + + return {item['url_id']: item for item in url_data} + + +def annotate_url(url, url_data): + '''Given a MirrorURL object, add a few more attributes to it regarding + status, including completion_pct, delay, and score.''' + known_attrs = ( + ('success_count', 0), + ('check_count', 0), + ('completion_pct', None), + ('last_check', None), + ('last_sync', None), + ('delay', None), + ('score', None), + ) + for k, v in known_attrs: + setattr(url, k, v) + for k, v in url_data.items(): + if k not in ('url_id', 'mirror_id'): + setattr(url, k, v) + + if url.check_count > 0: + url.completion_pct = float(url.success_count) / url.check_count + + if url.delay is not None: + hours = url.delay.days * 24.0 + url.delay.seconds / 3600.0 + + if url.completion_pct > 0: + divisor = url.completion_pct + else: + # arbitrary small value + divisor = 0.005 + stddev = url.duration_stddev or 0.0 + url.score = (hours + url.duration_avg + stddev) / divisor @cache_function(123) @@ -65,29 +122,14 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): if mirror_id: valid_urls = valid_urls.filter(mirror_id=mirror_id) - url_data = MirrorUrl.objects.values('id', 'mirror_id').filter( - id__in=valid_urls, logs__check_time__gte=cutoff_time).annotate( - check_count=Count('logs'), - success_count=Count('logs__duration'), - last_sync=Max('logs__last_sync'), - last_check=Max('logs__check_time'), - duration_avg=Avg('logs__duration')) - - vendor = database_vendor(MirrorUrl) - if vendor != 'sqlite': - url_data = url_data.annotate(duration_stddev=StdDev('logs__duration')) - + url_data = status_data(cutoff_time, mirror_id) urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( id__in=valid_urls).order_by('mirror__id', 'url') - delays = url_delays(cutoff_time, mirror_id) if urls: - url_data = dict((item['id'], item) for item in url_data) for url in urls: - for k, v in url_data.get(url.id, {}).items(): - if k not in ('id', 'mirror_id'): - setattr(url, k, v) - last_check = max([u.last_check for u in urls]) + annotate_url(url, url_data.get(url.id, {})) + last_check = max([u.last_check for u in urls if u.last_check]) num_checks = max([u.check_count for u in urls]) check_info = MirrorLog.objects.filter(check_time__gte=cutoff_time) if mirror_id: @@ -104,12 +146,6 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): num_checks = 0 check_frequency = None - for url in urls: - # fake the standard deviation for local testing setups - if vendor == 'sqlite': - setattr(url, 'duration_stddev', 0.0) - annotate_url(url, delays.get(url.id, None)) - return { 'cutoff': cutoff, 'last_check': last_check, -- cgit v1.1-4-g5e80 From f357a39a49a8edc713d512976a0be2a2a8ac5c4f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 14 Apr 2013 13:21:16 -0500 Subject: Remove cache_function decorator from a few spots The benefit of these storage operations might be outweighed by the cost, especially given how infrequently these functions are called. Signed-off-by: Dan McGee --- main/models.py | 5 +---- mirrors/utils.py | 2 -- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/main/models.py b/main/models.py index a561f4f6..89215f05 100644 --- a/main/models.py +++ b/main/models.py @@ -10,7 +10,7 @@ from django.contrib.sites.models import Site from django.utils.timezone import now from .fields import PositiveBigIntegerField -from .utils import cache_function, set_created_field +from .utils import set_created_field from devel.models import DeveloperKey from packages.alpm import AlpmAPI @@ -187,7 +187,6 @@ class Package(models.Model): self._applicable_arches = list(arches) return self._applicable_arches - #@cache_function(119) def get_requiredby(self): """ Returns a list of package objects. An attempt will be made to keep this @@ -265,7 +264,6 @@ class Package(models.Model): trimmed.append(dep) return trimmed - #@cache_function(121) def get_depends(self): """ Returns a list of dicts. Each dict contains ('dep', 'pkg', and @@ -293,7 +291,6 @@ class Package(models.Model): return (sort_order.get(dep.deptype, 1000), dep.name) return sorted(deps, key=sort_key) - #@cache_function(123) def reverse_conflicts(self): """ Returns a list of packages with conflicts against this package. diff --git a/mirrors/utils.py b/mirrors/utils.py index eb1211f1..5a8bbf5d 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -111,7 +111,6 @@ def annotate_url(url, url_data): url.score = (hours + url.duration_avg + stddev) / divisor -@cache_function(123) def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): cutoff_time = now() - cutoff @@ -155,7 +154,6 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): } -@cache_function(117) def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None): cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( -- cgit v1.1-4-g5e80 From 4d7d08f93de9e6af9e664a00e090158e738a890c Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 14 Apr 2013 13:45:00 -0500 Subject: Fix missing attribute error in replacment find code Whoops. Just introduced this when ensuring we look for both the packgae in other repositories as well as any replacments. Signed-off-by: Dan McGee --- packages/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/models.py b/packages/models.py index 92566a56..f830aade 100644 --- a/packages/models.py +++ b/packages/models.py @@ -326,7 +326,8 @@ class Update(models.Model): replaces__name=self.pkgname) if not self.arch.agnostic: # make sure we match architectures if possible - arches = self.pkg.applicable_arches() + arches = set(Arch.objects.filter(agnostic=True)) + arches.add(self.arch) pkgs = pkgs.filter(arch__in=arches) return pkgs -- cgit v1.1-4-g5e80 From 0cad22a5eca20ecb64b04d0912592ea6a5361e0d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 14 Apr 2013 13:59:49 -0500 Subject: Add a JSON view for retrieving mirror check locations Signed-off-by: Dan McGee --- mirrors/urls.py | 1 + mirrors/views.py | 33 +++++++++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/mirrors/urls.py b/mirrors/urls.py index 4e929410..7cf76aa1 100644 --- a/mirrors/urls.py +++ b/mirrors/urls.py @@ -6,6 +6,7 @@ urlpatterns = patterns('mirrors.views', (r'^status/json/$', 'status_json', {}, 'mirror-status-json'), (r'^status/tier/(?P\d+)/$', 'status', {}, 'mirror-status-tier'), (r'^status/tier/(?P\d+)/json/$', 'status_json', {}, 'mirror-status-tier-json'), + (r'^locations/json/$', 'locations_json', {}, 'mirror-locations-json'), (r'^(?P[\.\-\w]+)/$', 'mirror_details'), (r'^(?P[\.\-\w]+)/json/$', 'mirror_details_json'), ) diff --git a/mirrors/views.py b/mirrors/views.py index 07e28d40..30f96b63 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -13,7 +13,8 @@ from django.utils.timezone import now from django.views.decorators.csrf import csrf_exempt from django_countries.countries import COUNTRIES -from .models import Mirror, MirrorUrl, MirrorProtocol, MirrorLog +from .models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog, + CheckLocation) from .utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF COUNTRY_LOOKUP = dict(COUNTRIES) @@ -264,7 +265,8 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder): class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): '''Adds URL check history information.''' - log_attributes = ('check_time', 'last_sync', 'duration', 'is_success') + log_attributes = ('check_time', 'last_sync', 'duration', 'is_success', + 'location_id') def default(self, obj): if isinstance(obj, MirrorUrl): @@ -292,4 +294,31 @@ def status_json(request, tier=None): response = HttpResponse(to_json, content_type='application/json') return response + +class LocationJSONEncoder(DjangoJSONEncoder): + '''Base JSONEncoder extended to handle CheckLocation objects.''' + + def default(self, obj): + if hasattr(obj, '__iter__'): + # mainly for queryset serialization + return list(obj) + if isinstance(obj, CheckLocation): + return { + 'hostname': obj.hostname, + 'source_ip': obj.source_ip, + 'country': unicode(obj.country.name), + 'country_code': obj.country.code, + 'ip_version': obj.ip_version, + } + return super(LocationJSONEncoder, self).default(obj) + + +def locations_json(request): + data = {} + data['version'] = 1 + data['locations'] = CheckLocation.objects.all() + to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + # vim: set ts=4 sw=4 et: -- cgit v1.1-4-g5e80 From e4e88638b68ca34cc15ea5551f0e8a1a4c2e03d5 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 14 Apr 2013 14:01:52 -0500 Subject: Turn visualize-mirror into a CSS class --- sitestatic/archweb.css | 8 ++++---- templates/mirrors/mirror_details.html | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sitestatic/archweb.css b/sitestatic/archweb.css index a0dfb2ec..5c506914 100644 --- a/sitestatic/archweb.css +++ b/sitestatic/archweb.css @@ -885,19 +885,19 @@ table td.country { display: inline; } -#visualize-mirror .axis path, -#visualize-mirror .axis line { +.visualize-mirror .axis path, +.visualize-mirror .axis line { fill: none; stroke: #000; stroke-width: 3px; shape-rendering: crispEdges; } -#visualize-mirror .url-dot { +.visualize-mirror .url-dot { stroke: #000; } -#visualize-mirror .url-line { +.visualize-mirror .url-line { fill: none; stroke-width: 1.5px; } diff --git a/templates/mirrors/mirror_details.html b/templates/mirrors/mirror_details.html index 02d68a3a..ee60157b 100644 --- a/templates/mirrors/mirror_details.html +++ b/templates/mirrors/mirror_details.html @@ -109,7 +109,7 @@

Mirror Status Chart

-
+
{% load cdn %}{% jquery %}{% jquery_tablesorter %} -- cgit v1.1-4-g5e80 From 9b07cb1ebdc8c5cc5dff66a7edb02e0ddc9f4733 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 14 Apr 2013 15:08:25 -0500 Subject: Draw one mirror status graph per check location Rather than lump it all together and have odd spikes depending on which side of the Atlantic checked a mirror in a given timeslot, draw a chart per check location. Signed-off-by: Dan McGee --- mirrors/static/mirror_status.js | 68 +++++++++++++++++++++++++---------- mirrors/views.py | 3 +- templates/mirrors/mirror_details.html | 12 +++++-- 3 files changed, 60 insertions(+), 23 deletions(-) diff --git a/mirrors/static/mirror_status.js b/mirrors/static/mirror_status.js index 8ec85c40..4a57128a 100644 --- a/mirrors/static/mirror_status.js +++ b/mirrors/static/mirror_status.js @@ -1,7 +1,16 @@ -function mirror_status(chart_id, data_url) { - var jq_div = jQuery(chart_id); +function draw_graphs(location_url, log_url, container_id) { + $.when($.getJSON(location_url), $.getJSON(log_url)) + .then(function(loc_data, log_data) { + $.each(loc_data[0].locations, function(i, val) { + mirror_status(container_id, val, log_data[0]); + }); + }); +} - var draw_graph = function(data) { +function mirror_status(container_id, check_loc, log_data) { + + var draw_graph = function(chart_id, data) { + var jq_div = jQuery(chart_id); var margin = {top: 20, right: 20, bottom: 30, left: 40}, width = jq_div.width() - margin.left - margin.right, height = jq_div.height() - margin.top - margin.bottom; @@ -106,31 +115,52 @@ function mirror_status(chart_id, data_url) { .text(function(d) { return d; }); }; - /* invoke the data-fetch + first draw */ - var cached_data = null; - d3.json(data_url, function(json) { - cached_data = jQuery.map(json.urls, function(url, i) { + var filter_data = function(json, location_id) { + return jQuery.map(json.urls, function(url, i) { + var logs = jQuery.map(url.logs, function(log, j) { + if (!log.is_success) { + return null; + } + /* screen by location ID if we were given one */ + if (location_id && log.location_id !== location_id) { + return null; + } + return { + duration: log.duration, + check_time: new Date(log.check_time) + }; + }); + /* don't return URLs without any log info */ + if (logs.length === 0) { + return null; + } return { url: url.url, - logs: jQuery.map(url.logs, function(log, j) { - if (!log.is_success) { - return null; - } - return { - duration: log.duration, - check_time: new Date(log.check_time) - }; - }) + logs: logs }; }); - draw_graph(cached_data); - }); + }; + + var cached_data = filter_data(log_data, check_loc.id); + /* we had a check location with no log data handed to us, skip graphing */ + if (cached_data.length === 0) { + return; + } + + /* create the containers, defer the actual graph drawing */ + var chart_id = 'status-chart-' + check_loc.id; + $(container_id).append('

' + check_loc.country + ' (' + check_loc.source_ip + '), IPv' + check_loc.ip_version + '

'); + $(container_id).append('
'); + $(container_id).append('
'); + setTimeout(function() { + draw_graph('#' + chart_id, cached_data); + }, 0); /* then hook up a resize handler to redraw if necessary */ var resize_timeout = null; var real_resize = function() { resize_timeout = null; - draw_graph(cached_data); + draw_graph('#' + chart_id, cached_data); }; jQuery(window).resize(function() { if (resize_timeout) { diff --git a/mirrors/views.py b/mirrors/views.py index 30f96b63..9311fb8f 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -304,6 +304,7 @@ class LocationJSONEncoder(DjangoJSONEncoder): return list(obj) if isinstance(obj, CheckLocation): return { + 'id': obj.pk, 'hostname': obj.hostname, 'source_ip': obj.source_ip, 'country': unicode(obj.country.name), @@ -316,7 +317,7 @@ class LocationJSONEncoder(DjangoJSONEncoder): def locations_json(request): data = {} data['version'] = 1 - data['locations'] = CheckLocation.objects.all() + data['locations'] = CheckLocation.objects.all().order_by('pk') to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder) response = HttpResponse(to_json, content_type='application/json') return response diff --git a/templates/mirrors/mirror_details.html b/templates/mirrors/mirror_details.html index ee60157b..1c9a970e 100644 --- a/templates/mirrors/mirror_details.html +++ b/templates/mirrors/mirror_details.html @@ -106,10 +106,16 @@ {% endfor %} + + +
+

Mirror Status Charts

-

Mirror Status Chart

+

Periodic checks of the mirrors are done from various geographic + locations, IP addresses, and using IPv4 or IPv6. These results are + summarized in graphical form below.

-
+
{% load cdn %}{% jquery %}{% jquery_tablesorter %} @@ -122,7 +128,7 @@ $(document).ready(function() { headers: { 8: { sorter: 'mostlydigit' }, 9: { sorter: 'mostlydigit' }, 10: { sorter: 'mostlydigit' } } }); }); $(document).ready(function() { - mirror_status("#visualize-mirror", "./json/"); + draw_graphs("/mirrors/locations/json/", "./json/", "#charts-container"); }); {% endblock %} -- cgit v1.1-4-g5e80 From 4fd50fa622b13ecc5104919c0e7ed51f64734d92 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 14 Apr 2013 15:23:09 -0500 Subject: Tweaks to mirror status chart generation * Use 'jQuery' rather than '$' * Use same colors for URLs in every chart for clarity Signed-off-by: Dan McGee --- mirrors/static/mirror_status.js | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/mirrors/static/mirror_status.js b/mirrors/static/mirror_status.js index 4a57128a..241f5c61 100644 --- a/mirrors/static/mirror_status.js +++ b/mirrors/static/mirror_status.js @@ -1,13 +1,15 @@ function draw_graphs(location_url, log_url, container_id) { - $.when($.getJSON(location_url), $.getJSON(log_url)) + jQuery.when(jQuery.getJSON(location_url), jQuery.getJSON(log_url)) .then(function(loc_data, log_data) { - $.each(loc_data[0].locations, function(i, val) { - mirror_status(container_id, val, log_data[0]); + /* use the same color selection for a given URL in every graph */ + var color = d3.scale.category10(); + jQuery.each(loc_data[0].locations, function(i, val) { + mirror_status(container_id, val, log_data[0], color); }); }); } -function mirror_status(container_id, check_loc, log_data) { +function mirror_status(container_id, check_loc, log_data, color) { var draw_graph = function(chart_id, data) { var jq_div = jQuery(chart_id); @@ -15,8 +17,7 @@ function mirror_status(container_id, check_loc, log_data) { width = jq_div.width() - margin.left - margin.right, height = jq_div.height() - margin.top - margin.bottom; - var color = d3.scale.category10(), - x = d3.time.scale.utc().range([0, width]), + var x = d3.time.scale.utc().range([0, width]), y = d3.scale.linear().range([height, 0]), x_axis = d3.svg.axis().scale(x).orient("bottom"), y_axis = d3.svg.axis().scale(y).orient("left"); @@ -95,8 +96,9 @@ function mirror_status(container_id, check_loc, log_data) { .text(function(d) { return d.url + "\n" + d.duration.toFixed(3) + " secs\n" + d.check_time.toUTCString(); }); /* add a legend for good measure */ + var active = jQuery.map(data, function(item, i) { return item.url; }); var legend = svg.selectAll(".legend") - .data(color.domain()) + .data(active) .enter().append("g") .attr("class", "legend") .attr("transform", function(d, i) { return "translate(0," + i * 20 + ")"; }); @@ -149,9 +151,9 @@ function mirror_status(container_id, check_loc, log_data) { /* create the containers, defer the actual graph drawing */ var chart_id = 'status-chart-' + check_loc.id; - $(container_id).append('

' + check_loc.country + ' (' + check_loc.source_ip + '), IPv' + check_loc.ip_version + '

'); - $(container_id).append('
'); - $(container_id).append('
'); + jQuery(container_id).append('

' + check_loc.country + ' (' + check_loc.source_ip + '), IPv' + check_loc.ip_version + '

'); + jQuery(container_id).append('
'); + jQuery(container_id).append('
'); setTimeout(function() { draw_graph('#' + chart_id, cached_data); }, 0); -- cgit v1.1-4-g5e80 From 283cd944beefce8e364f238f25133e2d65b7702b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 16 Apr 2013 20:16:06 -0500 Subject: Use require_safe decorator rather than require_GET This was added in Django 1.4, and ensures both GET and HEAD requests, but not POST requests, are allowed through. Signed-off-by: Dan McGee --- packages/views/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/views/__init__.py b/packages/views/__init__.py index 4c195385..c1f0f492 100644 --- a/packages/views/__init__.py +++ b/packages/views/__init__.py @@ -9,7 +9,7 @@ from django.db.models import Q from django.http import HttpResponse from django.shortcuts import redirect, render from django.views.decorators.cache import cache_control -from django.views.decorators.http import require_GET, require_POST +from django.views.decorators.http import require_safe, require_POST from main.models import Package, Arch from ..models import PackageRelation @@ -24,7 +24,7 @@ from .search import search_json from .signoff import signoffs, signoff_package, signoff_options, signoffs_json -@require_GET +@require_safe @cache_control(public=True, max_age=86400) def opensearch(request): if request.is_secure(): @@ -37,7 +37,7 @@ def opensearch(request): content_type='application/opensearchdescription+xml') -@require_GET +@require_safe @cache_control(public=True, max_age=300) def opensearch_suggest(request): search_term = request.GET.get('q', '') -- cgit v1.1-4-g5e80 From 1f126744417f40c1a27e8d53054b7bc588d12a00 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 16 Apr 2013 21:51:18 -0500 Subject: Grab author when loading news item details Saves a query to the database. Signed-off-by: Dan McGee --- news/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/views.py b/news/views.py index 62d30fde..ca4fdf97 100644 --- a/news/views.py +++ b/news/views.py @@ -18,7 +18,7 @@ class NewsForm(forms.ModelForm): class NewsDetailView(DetailView): - model = News + queryset = News.objects.all().select_related('author') template_name = "news/view.html" -- cgit v1.1-4-g5e80 From 0589853360d12a1746e2d8e92e798f2727a0b5df Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 16 Apr 2013 21:52:20 -0500 Subject: Remove COUNTRY_LOOKUP global variable This is only used in one place, so it makes more sense for it to not be globally accessible. Signed-off-by: Dan McGee --- mirrors/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mirrors/views.py b/mirrors/views.py index 9311fb8f..73d40297 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -17,8 +17,6 @@ from .models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog, CheckLocation) from .utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF -COUNTRY_LOOKUP = dict(COUNTRIES) - class MirrorlistForm(forms.Form): country = forms.MultipleChoiceField(required=False) @@ -29,6 +27,8 @@ class MirrorlistForm(forms.Form): widget=CheckboxSelectMultiple) use_mirror_status = forms.BooleanField(required=False) + countries = dict(COUNTRIES) + def __init__(self, *args, **kwargs): super(MirrorlistForm, self).__init__(*args, **kwargs) fields = self.fields @@ -46,7 +46,7 @@ class MirrorlistForm(forms.Form): country_codes.update(MirrorUrl.objects.filter( mirror__active=True).exclude(country='').values_list( 'country', flat=True).order_by().distinct()) - countries = [(code, COUNTRY_LOOKUP[code]) for code in country_codes] + countries = [(code, self.countries[code]) for code in country_codes] return sorted(countries, key=itemgetter(1)) def as_div(self): -- cgit v1.1-4-g5e80 From 31d39e75eea7fb6cdf3bb8bfd8b490d45de04ee9 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 16 Apr 2013 21:59:32 -0500 Subject: Add shortcut for HEAD requests on slower views We sometimes see some web bots and crawlers make HEAD requests to verify existence of certain pages in the application. However, they are less than kind as 20-50 requests might arrive at the same time, and package search and details pages are some of the slowest rendering pages we have due to the Django template engine. Rather than waste time generating the content only to throw it away, response as soon as we can with either a 404 or 200 response as appropriate, omitting the 'Content-Length' header completely, which seems to be acceptable by the HTTP spec. Signed-off-by: Dan McGee --- main/utils.py | 9 +++++++++ packages/views/display.py | 3 +++ packages/views/search.py | 4 +++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/main/utils.py b/main/utils.py index cdd4ff71..8394e5cd 100644 --- a/main/utils.py +++ b/main/utils.py @@ -8,6 +8,7 @@ import hashlib from django.core.cache import cache from django.db import connections, router +from django.http import HttpResponse from django.utils.timezone import now from django.template.defaultfilters import slugify @@ -55,6 +56,14 @@ def clear_cache_function(func, args, kwargs): cache.delete(key) +def empty_response(): + empty = HttpResponse('') + # designating response as 'streaming' forces ConditionalGetMiddleware to + # not add a 'Content-Length: 0' header + empty.streaming = True + return empty + + def format_http_headers(request): headers = sorted((k, v) for k, v in request.META.items() if k.startswith('HTTP_')) diff --git a/packages/views/display.py b/packages/views/display.py index 50783835..87424483 100644 --- a/packages/views/display.py +++ b/packages/views/display.py @@ -7,6 +7,7 @@ from django.shortcuts import get_object_or_404, redirect, render from django.utils.timezone import now from main.models import Package, PackageFile, Arch, Repo +from main.utils import empty_response from mirrors.utils import get_mirror_url_for_download from ..models import Update from ..utils import get_group_info, PackageJSONEncoder @@ -126,6 +127,8 @@ def details(request, name='', repo='', arch=''): pkg = Package.objects.select_related( 'arch', 'repo', 'packager').get(pkgname=name, repo=repo_obj, arch=arch_obj) + if request.method == 'HEAD': + return empty_response() return render(request, 'packages/details.html', {'pkg': pkg}) except Package.DoesNotExist: # attempt a variety of fallback options before 404ing diff --git a/packages/views/search.py b/packages/views/search.py index 0362602e..b3778172 100644 --- a/packages/views/search.py +++ b/packages/views/search.py @@ -7,7 +7,7 @@ from django.http import HttpResponse from django.views.generic import ListView from main.models import Package, Arch, Repo -from main.utils import make_choice +from main.utils import empty_response, make_choice from ..models import PackageRelation from ..utils import attach_maintainers, PackageJSONEncoder @@ -99,6 +99,8 @@ class SearchListView(ListView): allowed_sort = list(sort_fields) + ["-" + s for s in sort_fields] def get(self, request, *args, **kwargs): + if request.method == 'HEAD': + return empty_response() self.form = PackageSearchForm(data=request.GET, show_staging=self.request.user.is_authenticated()) return super(SearchListView, self).get(request, *args, **kwargs) -- cgit v1.1-4-g5e80 From b7b24740640e24883cd17fd683e1d465fbb343f8 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 16 Apr 2013 22:12:01 -0500 Subject: Various minor code cleanups and fixes Most of these were suggested by PyCharm, and include everything from little syntax issues and other bad smells to dead or bad code. Signed-off-by: Dan McGee --- devel/management/commands/pgp_import.py | 1 + devel/models.py | 1 - devel/utils.py | 2 +- devel/views.py | 2 +- main/log.py | 1 - main/migrations/0029_fill_in_repo_data.py | 1 - main/models.py | 12 +++++------- main/utils.py | 1 - mirrors/management/commands/mirrorcheck.py | 12 +++--------- mirrors/models.py | 2 +- mirrors/utils.py | 6 +++--- packages/migrations/0002_populate_package_relation.py | 2 -- packages/templatetags/package_extras.py | 4 ++-- packages/utils.py | 2 +- packages/views/display.py | 2 -- packages/views/flag.py | 3 +-- public/views.py | 1 - releng/management/commands/syncisos.py | 2 +- releng/models.py | 2 +- releng/views.py | 2 +- retro/templates/retro/index-20030330.html | 1 - sitestatic/archweb.js | 1 - todolists/utils.py | 1 - todolists/views.py | 1 - visualize/static/visualize.js | 2 +- 25 files changed, 23 insertions(+), 44 deletions(-) diff --git a/devel/management/commands/pgp_import.py b/devel/management/commands/pgp_import.py index 10e6cfcb..b1f29d77 100644 --- a/devel/management/commands/pgp_import.py +++ b/devel/management/commands/pgp_import.py @@ -95,6 +95,7 @@ def parse_keydata(data): # parse all of the output from our successful GPG command logger.info("parsing command output") + node = None for line in data.split('\n'): parts = line.split(':') if parts[0] == 'pub': diff --git a/devel/models.py b/devel/models.py index 67de40a6..4354e0f2 100644 --- a/devel/models.py +++ b/devel/models.py @@ -4,7 +4,6 @@ import pytz from django.db import models from django.db.models.signals import pre_save from django.contrib.auth.models import User -from django.utils.timezone import now from django_countries import CountryField from .fields import PGPKeyField diff --git a/devel/utils.py b/devel/utils.py index e8e3a6c4..340841f5 100644 --- a/devel/utils.py +++ b/devel/utils.py @@ -131,7 +131,7 @@ class UserFinder(object): self.username_email, self.user_name) for matcher in find_methods: user = matcher(name, email) - if user != None: + if user is not None: break self.cache[userstring] = user diff --git a/devel/views.py b/devel/views.py index 61c1e568..4258ea7f 100644 --- a/devel/views.py +++ b/devel/views.py @@ -34,7 +34,7 @@ from .utils import get_annotated_maintainers @login_required def index(request): '''the developer dashboard''' - if(request.user.is_authenticated()): + if request.user.is_authenticated(): inner_q = PackageRelation.objects.filter(user=request.user) else: inner_q = PackageRelation.objects.none() diff --git a/main/log.py b/main/log.py index 63634874..5c745cc8 100644 --- a/main/log.py +++ b/main/log.py @@ -46,7 +46,6 @@ class RateLimitFilter(object): trace = '\n'.join(traceback.format_exception(*record.exc_info)) key = md5(trace).hexdigest() - duplicate = False cache = self.cache_module.cache # Test if the cache works diff --git a/main/migrations/0029_fill_in_repo_data.py b/main/migrations/0029_fill_in_repo_data.py index 0887b28c..7da6b1c4 100644 --- a/main/migrations/0029_fill_in_repo_data.py +++ b/main/migrations/0029_fill_in_repo_data.py @@ -7,7 +7,6 @@ from django.db import models class Migration(DataMigration): def forwards(self, orm): - "Write your forwards methods here." orm.Repo.objects.filter(name__istartswith='community').update(bugs_project=5, svn_root='community') orm.Repo.objects.filter(name__iexact='multilib').update(bugs_project=5, svn_root='community') diff --git a/main/models.py b/main/models.py index 89215f05..24aeed89 100644 --- a/main/models.py +++ b/main/models.py @@ -7,7 +7,6 @@ from django.db import models from django.db.models import Q from django.contrib.auth.models import User from django.contrib.sites.models import Site -from django.utils.timezone import now from .fields import PositiveBigIntegerField from .utils import set_created_field @@ -140,7 +139,7 @@ class Package(models.Model): @property def signature(self): try: - data = b64decode(self.pgp_signature) + data = b64decode(self.pgp_signature.encode('utf-8')) except TypeError: return None if not data: @@ -274,7 +273,6 @@ class Package(models.Model): Packages will match the testing status of this package if possible. """ deps = [] - arches = None # TODO: we can use list comprehension and an 'in' query to make this # more effective for dep in self.depends.all(): @@ -400,13 +398,13 @@ class Package(models.Model): '''attempt to locate this package anywhere else, regardless of architecture or repository. Excludes this package from the list.''' names = [self.pkgname] - if self.pkgname.startswith('lib32-'): + if self.pkgname.startswith(u'lib32-'): names.append(self.pkgname[6:]) - elif self.pkgname.endswith('-multilib'): + elif self.pkgname.endswith(u'-multilib'): names.append(self.pkgname[:-9]) else: - names.append('lib32-' + self.pkgname) - names.append(self.pkgname + '-multilib') + names.append(u'lib32-' + self.pkgname) + names.append(self.pkgname + u'-multilib') return Package.objects.normal().filter( pkgname__in=names).exclude(id=self.id).order_by( 'arch__name', 'repo__name') diff --git a/main/utils.py b/main/utils.py index 8394e5cd..9ee8db58 100644 --- a/main/utils.py +++ b/main/utils.py @@ -3,7 +3,6 @@ try: except ImportError: import pickle -from datetime import datetime import hashlib from django.core.cache import cache diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index d6de8f22..e7dd7b49 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -106,19 +106,13 @@ def parse_lastsync(log, data): def check_mirror_url(mirror_url, location, timeout): - if location: - if location.family == socket.AF_INET6: - ipopt = '--ipv6' - elif location.family == socket.AF_INET: - ipopt = '--ipv4' - url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) log = MirrorLog(url=mirror_url, check_time=now(), location=location) headers = {'User-Agent': 'archweb/1.0'} req = urllib2.Request(url, None, headers) + start = time.time() try: - start = time.time() result = urllib2.urlopen(req, timeout=timeout) data = result.read() result.close() @@ -147,12 +141,12 @@ def check_mirror_url(mirror_url, location, timeout): elif isinstance(e.reason, socket.error): log.error = e.reason.args[1] logger.debug("failed: %s, %s", url, log.error) - except HTTPException as e: + except HTTPException: # e.g., BadStatusLine log.is_success = False log.error = "Exception in processing HTTP request." logger.debug("failed: %s, %s", url, log.error) - except socket.timeout as e: + except socket.timeout: log.is_success = False log.error = "Connection timed out." logger.debug("failed: %s, %s", url, log.error) diff --git a/mirrors/models.py b/mirrors/models.py index 791b0078..d8ac7952 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -92,7 +92,7 @@ class MirrorUrl(models.Model): families = self.address_families() self.has_ipv4 = socket.AF_INET in families self.has_ipv6 = socket.AF_INET6 in families - except socket.error as e: + except socket.error: # We don't fail in this case; we'll just set both to False self.has_ipv4 = False self.has_ipv6 = False diff --git a/mirrors/utils.py b/mirrors/utils.py index 5a8bbf5d..531cf005 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -1,13 +1,13 @@ from datetime import timedelta from django.db import connection -from django.db.models import Avg, Count, Max, Min, StdDev +from django.db.models import Count, Max, Min from django.utils.dateparse import parse_datetime from django.utils.timezone import now from django_countries.fields import Country from main.utils import cache_function, database_vendor -from .models import MirrorLog, MirrorProtocol, MirrorUrl +from .models import MirrorLog, MirrorUrl DEFAULT_CUTOFF = timedelta(hours=24) @@ -165,7 +165,7 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None): ).order_by('-last_occurred', '-error_count') if mirror_id: - urls = urls.filter(mirror_id=mirror_id) + errors = errors.filter(url__mirror_id=mirror_id) errors = list(errors) for err in errors: diff --git a/packages/migrations/0002_populate_package_relation.py b/packages/migrations/0002_populate_package_relation.py index 738e068f..b0d32c7a 100644 --- a/packages/migrations/0002_populate_package_relation.py +++ b/packages/migrations/0002_populate_package_relation.py @@ -11,7 +11,6 @@ class Migration(DataMigration): ) def forwards(self, orm): - "Write your forwards methods here." # search by pkgbase first and insert those records qs = orm['main.Package'].objects.exclude(maintainer=None).exclude( pkgbase=None).distinct().values('pkgbase', 'maintainer_id') @@ -29,7 +28,6 @@ class Migration(DataMigration): defaults={'user_id': row['maintainer_id']}) def backwards(self, orm): - "Write your backwards methods here." if not db.dry_run: orm.PackageRelation.objects.all().delete() pass diff --git a/packages/templatetags/package_extras.py b/packages/templatetags/package_extras.py index f14fab1e..ef0e1aea 100644 --- a/packages/templatetags/package_extras.py +++ b/packages/templatetags/package_extras.py @@ -53,10 +53,10 @@ def do_buildsortqs(parser, token): tagname, sortfield = token.split_contents() except ValueError: raise template.TemplateSyntaxError( - "%r tag requires a single argument" % tagname) + "%r tag requires a single argument" % token) if not (sortfield[0] == sortfield[-1] and sortfield[0] in ('"', "'")): raise template.TemplateSyntaxError( - "%r tag's argument should be in quotes" % tagname) + "%r tag's argument should be in quotes" % token) return BuildQueryStringNode(sortfield[1:-1]) diff --git a/packages/utils.py b/packages/utils.py index a4217fbd..4f3b8665 100644 --- a/packages/utils.py +++ b/packages/utils.py @@ -391,7 +391,7 @@ SELECT DISTINCT s.id """ cursor = connection.cursor() # query pre-process- fill in table name and placeholders for IN - repo_sql = ','.join(['%s' for r in repos]) + repo_sql = ','.join(['%s' for _ in repos]) sql = sql % (model._meta.db_table, repo_sql, repo_sql) repo_ids = [r.pk for r in repos] # repo_ids are needed twice, so double the array diff --git a/packages/views/display.py b/packages/views/display.py index 87424483..021c7ed8 100644 --- a/packages/views/display.py +++ b/packages/views/display.py @@ -228,8 +228,6 @@ def download(request, name, repo, arch): if pkg.arch.agnostic: # grab the first non-any arch to fake the download path arch = Arch.objects.exclude(agnostic=True)[0].name - values = { - } url = '{host}{repo}/os/{arch}/{filename}'.format(host=url.url, repo=pkg.repo.name.lower(), arch=arch, filename=pkg.filename) return redirect(url) diff --git a/packages/views/flag.py b/packages/views/flag.py index 5c76e1d5..39cdcef8 100644 --- a/packages/views/flag.py +++ b/packages/views/flag.py @@ -110,7 +110,7 @@ def flag(request, name, repo, arch): subject = '%s package [%s] marked out-of-date' % \ (pkg.repo.name, pkg.pkgname) for maint in maints: - if maint.userprofile.notify == True: + if maint.userprofile.notify is True: toemail.append(maint.email) if toemail: @@ -133,7 +133,6 @@ def flag(request, name, repo, arch): return redirect('package-flag-confirmed', name=name, repo=repo, arch=arch) else: - initial = {} form = FlagForm(authenticated=authenticated) context = { diff --git a/public/views.py b/public/views.py index 22cb8759..39273396 100644 --- a/public/views.py +++ b/public/views.py @@ -125,7 +125,6 @@ def keys(request): master_keys = MasterKey.objects.select_related('owner', 'revoker', 'owner__userprofile', 'revoker__userprofile').filter( revoked__isnull=True) - master_key_ids = frozenset(key.pgp_key[-16:] for key in master_keys) sig_counts = PGPSignature.objects.filter(not_expired, valid=True, signee__in=user_key_ids).order_by().values_list('signer').annotate( diff --git a/releng/management/commands/syncisos.py b/releng/management/commands/syncisos.py index c9f61964..f182cc33 100644 --- a/releng/management/commands/syncisos.py +++ b/releng/management/commands/syncisos.py @@ -20,7 +20,7 @@ class IsoListParser(HTMLParser): if tag == 'a': for name, value in attrs: if name == "href": - if value != '../' and self.url_re.search(value) != None: + if value != '../' and self.url_re.search(value) is not None: self.hyperlinks.append(value[:-1]) def parse(self, url): diff --git a/releng/models.py b/releng/models.py index b95f7d52..5ee2f325 100644 --- a/releng/models.py +++ b/releng/models.py @@ -160,7 +160,7 @@ class Release(models.Model): def torrent(self): try: - data = b64decode(self.torrent_data) + data = b64decode(self.torrent_data.encode('utf-8')) except TypeError: return None if not data: diff --git a/releng/views.py b/releng/views.py index ad4b07d1..b1c76a4a 100644 --- a/releng/views.py +++ b/releng/views.py @@ -231,7 +231,7 @@ def release_torrent(request, version): release = get_object_or_404(Release, version=version) if not release.torrent_data: raise Http404 - data = b64decode(release.torrent_data) + data = b64decode(release.torrent_data.encode('utf-8')) response = HttpResponse(data, content_type='application/x-bittorrent') # TODO: this is duplicated from Release.iso_url() filename = 'archlinux-%s-dual.iso.torrent' % release.version diff --git a/retro/templates/retro/index-20030330.html b/retro/templates/retro/index-20030330.html index 449731af..51cc8ba3 100644 --- a/retro/templates/retro/index-20030330.html +++ b/retro/templates/retro/index-20030330.html @@ -232,7 +232,6 @@ Happy holidays!
[ Older News ]

-


diff --git a/sitestatic/archweb.js b/sitestatic/archweb.js index dda22d9e..aa225f5f 100644 --- a/sitestatic/archweb.js +++ b/sitestatic/archweb.js @@ -146,7 +146,6 @@ if (typeof $ !== 'undefined' && typeof $.tablesorter !== 'undefined') { (function($) { $.fn.enableCheckboxRangeSelection = function() { var lastCheckbox = null, - lastElement = null, spec = this; spec.unbind("click.checkboxrange"); diff --git a/todolists/utils.py b/todolists/utils.py index 51a75a3c..7b98c887 100644 --- a/todolists/utils.py +++ b/todolists/utils.py @@ -1,5 +1,4 @@ from django.db import connections, router -from django.db.models import Count from .models import Todolist, TodolistPackage from packages.models import Package diff --git a/todolists/views.py b/todolists/views.py index 7636d38e..d5b39934 100644 --- a/todolists/views.py +++ b/todolists/views.py @@ -9,7 +9,6 @@ from django.db import transaction from django.views.decorators.cache import never_cache from django.views.generic import DeleteView from django.template import Context, loader -from django.template.defaultfilters import slugify from django.utils.timezone import now from main.models import Package, Repo diff --git a/visualize/static/visualize.js b/visualize/static/visualize.js index 7e240d44..5004fe6c 100644 --- a/visualize/static/visualize.js +++ b/visualize/static/visualize.js @@ -55,7 +55,7 @@ function packages_treemap(chart_id, orderings, default_order) { var nodes = d3_div.data([json]).selectAll("div") .data(treemap.nodes, key_func); /* start out new nodes in the center of the picture area */ - var w_center = jq_div.width() / 2; + var w_center = jq_div.width() / 2, h_center = jq_div.height() / 2; nodes.enter().append("div") .attr("class", "treemap-cell") -- cgit v1.1-4-g5e80 From 81821d2b4791f18635b13394aa57fa0795acddaf Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 16 Apr 2013 23:24:33 -0500 Subject: Sort required by entries by dependency type We were doing this for depends entries already, and we can do the same for required by entries via some absolutely awesome SQL query magic and injecting a few more things to our already crazy query. Signed-off-by: Dan McGee --- main/models.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/main/models.py b/main/models.py index 24aeed89..2f4d3520 100644 --- a/main/models.py +++ b/main/models.py @@ -193,14 +193,22 @@ class Package(models.Model): category as this package if that check makes sense. """ from packages.models import Depend + sorttype = '''(CASE deptype + WHEN 'D' THEN 0 + WHEN 'O' THEN 1 + WHEN 'M' THEN 2 + WHEN 'C' THEN 3 + ELSE 1000 END)''' name_clause = '''packages_depend.name IN ( SELECT %s UNION ALL SELECT z.name FROM packages_provision z WHERE z.pkg_id = %s )''' requiredby = Depend.objects.select_related('pkg', 'pkg__arch', 'pkg__repo').extra( - where=[name_clause], params=[self.pkgname, self.id]).order_by( - 'pkg__pkgname', 'pkg__arch__name', 'pkg__repo__name') + select={'sorttype': sorttype}, + where=[name_clause], params=[self.pkgname, self.id]).order_by( + 'sorttype', 'pkg__pkgname', + 'pkg__arch__name', 'pkg__repo__name') if not self.arch.agnostic: # make sure we match architectures if possible requiredby = requiredby.filter( -- cgit v1.1-4-g5e80 From 95ceb2a4a42093c9fd728e9e3d546943bc7b0b9d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 20 Apr 2013 11:03:00 -0500 Subject: Remove spaceless from depends and required by template fragments This was a bit of premature optimization, and ends up slowing the page generation a good amount since we have to run this ~400 times on some packages with long depends or required by lists. Webserver compression should handle this just fine and not result in too much page bloat. Signed-off-by: Dan McGee --- templates/packages/details_depend.html | 5 ++--- templates/packages/details_requiredby.html | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/templates/packages/details_depend.html b/templates/packages/details_depend.html index 4aa739c2..b89ffbfa 100644 --- a/templates/packages/details_depend.html +++ b/templates/packages/details_depend.html @@ -1,5 +1,4 @@ -{% load package_extras %}{% spaceless %}
  • -{% ifequal depend.pkg None %} +{% load package_extras %}
  • {% ifequal depend.pkg None %} {% if depend.providers %}{{ depend.dep.name }}{{ depend.dep.comparison|default:"" }}{{ depend.dep.version|default:"" }} ({% multi_pkg_details depend.providers %}) {% else %}{{ depend.dep.name }}{{ depend.dep.comparison|default:"" }}{{ depend.dep.version|default:"" }} (virtual) {% endif %}{% else %} @@ -11,4 +10,4 @@ {% endif %}{% if depend.dep.deptype == 'M' %} (make) {% endif %}{% if depend.dep.deptype == 'C' %} (check) {% endif %}{% if depend.dep.description %} - {{ depend.dep.description }} -{% endif %}
    • {% endspaceless %} +{% endif %} diff --git a/templates/packages/details_requiredby.html b/templates/packages/details_requiredby.html index e8c713ac..504a322f 100644 --- a/templates/packages/details_requiredby.html +++ b/templates/packages/details_requiredby.html @@ -1,8 +1,8 @@ -{% load package_extras %}{% spaceless %}
  • {% pkg_details_link req.pkg %} +{% load package_extras %}
  • {% pkg_details_link req.pkg %} {% if req.name != pkg.pkgname %} (requires {{ req.name }}) {% endif %}{% if req.pkg.repo.testing %} (testing) {% endif %}{% if req.pkg.repo.staging %} (staging) {% endif %}{% if req.deptype == 'O' %} (optional) {% endif %}{% if req.deptype == 'M' %} (make) {% endif %}{% if req.deptype == 'C' %} (check) -{% endif %}
    • {% endspaceless %} +{% endif %} -- cgit v1.1-4-g5e80 From b519c40e0af5b86ba660cf9d9cfc53a66fd8bef8 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 20 Apr 2013 11:11:04 -0500 Subject: Remove {% spaceless %} tag from a few more places This tag is simply not worth the time it takes to do what it does. Signed-off-by: Dan McGee --- templates/mirrors/status.html | 6 ++---- templates/mirrors/status_table.html | 6 ++---- templates/packages/search.html | 6 ++---- templates/packages/signoffs.html | 8 ++------ 4 files changed, 8 insertions(+), 18 deletions(-) diff --git a/templates/mirrors/status.html b/templates/mirrors/status.html index 8d32d3fa..26e81e79 100644 --- a/templates/mirrors/status.html +++ b/templates/mirrors/status.html @@ -87,16 +87,14 @@ - {% for log in error_logs %} - {% spaceless %} + {% for log in error_logs %} {{ log.url__url }} {{ log.url__protocol__protocol }} {% country_flag log.country %}{{ log.country.name }} {{ log.error|linebreaksbr }} {{ log.last_occurred|date:'Y-m-d H:i' }} {{ log.error_count }} - - {% endspaceless %}{% endfor %} + {% endfor %} diff --git a/templates/mirrors/status_table.html b/templates/mirrors/status_table.html index 2dd7ef49..e848a9c9 100644 --- a/templates/mirrors/status_table.html +++ b/templates/mirrors/status_table.html @@ -14,8 +14,7 @@ - {% for m_url in urls %} - {% spaceless %} + {% for m_url in urls %} {{ m_url.url }} {{ m_url.protocol }} {% country_flag m_url.country %}{{ m_url.country.name }} @@ -24,7 +23,6 @@ {{ m_url.duration_avg|floatformat:2 }} {{ m_url.duration_stddev|floatformat:2 }} {{ m_url.score|floatformat:1|default:'∞' }} - - {% endspaceless %}{% endfor %} + {% endfor %} diff --git a/templates/packages/search.html b/templates/packages/search.html index bf1eecd9..f50bc8be 100644 --- a/templates/packages/search.html +++ b/templates/packages/search.html @@ -69,8 +69,7 @@ - {% for pkg in package_list %} - {% spaceless %} + {% for pkg in package_list %} {% if perms.main.change_package %} {% endif %} @@ -85,8 +84,7 @@ {{ pkg.pkgdesc }} {{ pkg.last_update|date }} {{ pkg.flag_date|date }} - - {% endspaceless %}{% endfor %} + {% endfor %} {% include "packages/search_paginator.html" %} diff --git a/templates/packages/signoffs.html b/templates/packages/signoffs.html index 20371f95..2879c30b 100644 --- a/templates/packages/signoffs.html +++ b/templates/packages/signoffs.html @@ -50,9 +50,7 @@ - {% for group in signoff_groups %} - - {% spaceless %} + {% for group in signoff_groups %} {% pkg_details_link group.package %} {{ group.version }} {{ group.arch.name }} {{ group.target_repo }} @@ -75,9 +73,7 @@ {% endcomment %}{% if spec.known_bad %}Package is known to be bad
    {% endif %}{% comment %} {% endcomment %}{{ spec.comments|default:""|linebreaksbr }} {% endwith %}{% endif %} - {% endspaceless %} - - {% endfor %} + {% endfor %} -- cgit v1.1-4-g5e80 From 6de0cfbd23aae69036439db817cc26740d8796cd Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 20 Apr 2013 11:13:49 -0500 Subject: Fix some None issues with sqlite3 and mirror status If certain attributes came back from the database as NULL, we had issues parsing them. Pass None/NULL straight through rather than trying to type-convert. Signed-off-by: Dan McGee --- mirrors/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mirrors/utils.py b/mirrors/utils.py index 531cf005..ba45da5f 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -71,8 +71,10 @@ GROUP BY l.url_id, u.mirror_id # sqlite loves to return less than ideal types if vendor == 'sqlite': for item in url_data: - item['delay'] = timedelta(seconds=item['delay']) - item['last_sync'] = parse_datetime(item['last_sync']) + if item['delay'] is not None: + item['delay'] = timedelta(seconds=item['delay']) + if item['last_sync'] is not None: + item['last_sync'] = parse_datetime(item['last_sync']) item['last_check'] = parse_datetime(item['last_check']) return {item['url_id']: item for item in url_data} -- cgit v1.1-4-g5e80 From 3e085183bf39c59fdcd0b9e67b5a02653a51d30e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 20 Apr 2013 11:28:12 -0500 Subject: Move all script blocks into {% script_block %} section I added this a while back, but didn't roll it out to all templates. Signed-off-by: Dan McGee --- templates/devel/clock.html | 3 +++ templates/devel/index.html | 2 ++ templates/devel/packages.html | 3 +++ templates/devel/profile.html | 2 ++ templates/mirrors/mirror_details.html | 3 +++ templates/mirrors/status.html | 3 +++ templates/packages/details.html | 2 ++ templates/packages/differences.html | 2 ++ templates/packages/groups.html | 3 +++ templates/packages/packages_list.html | 3 +++ templates/packages/signoffs.html | 3 +++ templates/packages/stale_relations.html | 3 +++ templates/public/keys.html | 2 ++ templates/registration/login.html | 2 ++ templates/releng/iso_overview.html | 3 +++ templates/releng/release_list.html | 2 ++ templates/releng/result_list.html | 3 +++ templates/todolists/list.html | 3 +++ templates/todolists/view.html | 3 +++ templates/visualize/index.html | 2 ++ 20 files changed, 52 insertions(+) diff --git a/templates/devel/clock.html b/templates/devel/clock.html index 83fbb70b..2c5bfacf 100644 --- a/templates/devel/clock.html +++ b/templates/devel/clock.html @@ -56,6 +56,9 @@ +{% endblock %} + +{% block script_block %} {% load cdn %}{% jquery %}{% jquery_tablesorter %} diff --git a/templates/mirrors/status.html b/templates/mirrors/status.html index 26e81e79..283ff681 100644 --- a/templates/mirrors/status.html +++ b/templates/mirrors/status.html @@ -99,6 +99,9 @@ +{% endblock %} + +{% block script_block %} {% load cdn %}{% jquery %}{% jquery_tablesorter %} diff --git a/templates/registration/login.html b/templates/registration/login.html index edbb77ea..82f74f58 100644 --- a/templates/registration/login.html +++ b/templates/registration/login.html @@ -21,7 +21,9 @@ +{% endblock %} +{% block script_block %} {% load cdn %}{% jquery %} -- cgit v1.1-4-g5e80 From 1364f689fd177e11ab04f0842035175baf80de35 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 20 Apr 2013 11:34:12 -0500 Subject: Minor CSS cleanups/tweaks Signed-off-by: Dan McGee --- sitestatic/archweb.css | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sitestatic/archweb.css b/sitestatic/archweb.css index 5c506914..b7d6e1ee 100644 --- a/sitestatic/archweb.css +++ b/sitestatic/archweb.css @@ -314,7 +314,7 @@ dl { dl dt, dl dd { margin-bottom: 4px; - padding: 8px 0px 4px; + padding: 8px 0 4px; font-weight: bold; border-top: 1px dotted #bbb; } @@ -420,7 +420,7 @@ table thead th.tablesorter-headerDesc { } table thead th.sorter-false { - background-image: url(); + background-image: none; cursor: default; } @@ -490,13 +490,13 @@ table thead th.sorter-false { h3 span.arrow { display: block; - width: 0px; - height: 0px; + width: 0; + height: 0; border-left: 6px solid transparent; border-right: 6px solid transparent; border-top: 6px solid #1794D1; margin: 0 auto; - font-size: 0px; + font-size: 0; line-height: 0px; } -- cgit v1.1-4-g5e80 From 7873cb9a052c7991f3d91ddf4dc05caef6305cd7 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 20 Apr 2013 13:31:20 -0500 Subject: Unbreak the package differences page We had a weird conditional around everything on the page that doesn't really need to be there. Remove it. Signed-off-by: Dan McGee --- templates/packages/differences.html | 2 -- 1 file changed, 2 deletions(-) diff --git a/templates/packages/differences.html b/templates/packages/differences.html index 3d3f6edf..b0b9b419 100644 --- a/templates/packages/differences.html +++ b/templates/packages/differences.html @@ -5,7 +5,6 @@ {% block navbarclass %}anb-packages{% endblock %} {% block content %} -{% if differences %}

    Package Differences by Architecture

    @@ -111,5 +110,4 @@ $(document).ready(function() { filter_packages(); }); -{% endif %} {% endblock %} -- cgit v1.1-4-g5e80 From bb18fa3323a0494a2774ea61911572b089d04b6d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 20 Apr 2013 13:32:09 -0500 Subject: Fix parsing issues when query string keys contain unicode This is dirty, but it works. There is probably a better and cleaner way to do all of this, but for now just fix it quickly. Signed-off-by: Dan McGee --- packages/templatetags/package_extras.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/templatetags/package_extras.py b/packages/templatetags/package_extras.py index ef0e1aea..f7392a96 100644 --- a/packages/templatetags/package_extras.py +++ b/packages/templatetags/package_extras.py @@ -37,6 +37,12 @@ class BuildQueryStringNode(template.Node): def render(self, context): qs = parse_qs(context['current_query']) + # This is really dirty. The crazy round trips we do on our query string + # mean we get things like u'\xe2\x98\x83' in our views, when we should + # have simply u'\u2603' or a byte string of the UTF-8 value. Force the + # keys and list of values to be byte strings only. + qs = {k.encode('latin-1'): [v.encode('latin-1') for v in vals] + for k, vals in qs.items()} if 'sort' in qs and self.sortfield in qs['sort']: if self.sortfield.startswith('-'): qs['sort'] = [self.sortfield[1:]] -- cgit v1.1-4-g5e80