From 9adc2e53124daa6d13090166830396ffff9013d3 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 27 Nov 2013 17:49:58 -0500 Subject: Stop using Django-provided floatformat template tag It turns out this is a HUGE part of our slow mirror status template rendering, due to the internal workings. Everything is converted to a Python decimal object which is way slower than just staying in native floating point. Given we are always dealing with floats when we need to do our formatting, a home-rolled template tag can accomplish this much faster. Signed-off-by: Dan McGee --- mirrors/templatetags/mirror_status.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'mirrors') diff --git a/mirrors/templatetags/mirror_status.py b/mirrors/templatetags/mirror_status.py index 9a363fbe..b3810d9a 100644 --- a/mirrors/templatetags/mirror_status.py +++ b/mirrors/templatetags/mirror_status.py @@ -1,6 +1,5 @@ from datetime import timedelta from django import template -from django.template.defaultfilters import floatformat register = template.Library() @@ -27,10 +26,16 @@ def hours(value): return '%d hours' % hrs @register.filter -def percentage(value, arg=-1): +def floatvalue(value, arg=2): + if value is None: + return u'' + return '%.*f' % (arg, value) + +@register.filter +def percentage(value, arg=1): if not value and type(value) != float: return u'' new_val = value * 100.0 - return floatformat(new_val, arg) + '%' + return '%.*f%%' % (arg, new_val) # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From ecece25814042d262bb7a102b9cbe48fc9c87db4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 1 Dec 2013 11:42:25 -0600 Subject: Show all mirror status data to authorized users Regardless of whether the mirror URL is active or not, we often have data we can show the end user, especially if mirror admins care to see the data we've been gathering. Signed-off-by: Dan McGee --- mirrors/utils.py | 20 +++++++++++--------- mirrors/views.py | 18 +++++++++++++----- 2 files changed, 24 insertions(+), 14 deletions(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index 633731a7..9f40bca6 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -113,18 +113,17 @@ def annotate_url(url, url_data): url.score = (hours + url.duration_avg + stddev) / divisor -def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): +def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None, show_all=False): cutoff_time = now() - cutoff - # TODO: this prevents grabbing data points from any mirror that was active, - # receiving checks, and then marked private. we can probably be smarter and - # filter the data later? - valid_urls = MirrorUrl.objects.filter(active=True, - mirror__active=True, mirror__public=True, + valid_urls = MirrorUrl.objects.filter( logs__check_time__gte=cutoff_time).distinct() if mirror_id: valid_urls = valid_urls.filter(mirror_id=mirror_id) + if not show_all: + valid_urls = valid_urls.filter(active=True, mirror__active=True, + mirror__public=True) url_data = status_data(cutoff_time, mirror_id) urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( @@ -159,11 +158,11 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): } -def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None): +def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None, show_all=False): cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( - is_success=False, check_time__gte=cutoff_time, url__active=True, - url__mirror__active=True, url__mirror__public=True).values( + is_success=False, check_time__gte=cutoff_time, + url__mirror__public=True).values( 'url__url', 'url__country', 'url__protocol__protocol', 'url__mirror__tier', 'error').annotate( error_count=Count('error'), last_occurred=Max('check_time') @@ -171,6 +170,9 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None): if mirror_id: errors = errors.filter(url__mirror_id=mirror_id) + if not show_all: + errors = errors.filter(url__active=True, url__mirror__active=True, + url__mirror__public=True) errors = list(errors) for err in errors: diff --git a/mirrors/views.py b/mirrors/views.py index ec056696..9e05e5fc 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -153,15 +153,20 @@ def mirrors(request): def mirror_details(request, name): mirror = get_object_or_404(Mirror, name=name) - if not request.user.is_authenticated() and \ + authorized = request.user.is_authenticated() + if not authorized and \ (not mirror.public or not mirror.active): raise Http404 error_cutoff = timedelta(days=7) - status_info = get_mirror_statuses(mirror_id=mirror.id) + status_info = get_mirror_statuses(mirror_id=mirror.id, + show_all=authorized) checked_urls = {url for url in status_info['urls'] \ if url.mirror_id == mirror.id} - all_urls = set(mirror.urls.filter(active=True).select_related('protocol')) + all_urls = mirror.urls.select_related('protocol') + if not authorized: + all_urls = all_urls.filter(active=True) + all_urls = set(all_urls) # Add dummy data for URLs that we haven't checked recently other_urls = all_urls.difference(checked_urls) for url in other_urls: @@ -170,7 +175,8 @@ def mirror_details(request, name): setattr(url, attr, None) all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url')) - error_logs = get_mirror_errors(mirror_id=mirror.id, cutoff=error_cutoff) + error_logs = get_mirror_errors(mirror_id=mirror.id, cutoff=error_cutoff, + show_all=True) context = { 'mirror': mirror, @@ -181,8 +187,10 @@ def mirror_details(request, name): return render(request, 'mirrors/mirror_details.html', context) def mirror_details_json(request, name): + authorized = request.user.is_authenticated() mirror = get_object_or_404(Mirror, name=name) - status_info = get_mirror_statuses(mirror_id=mirror.id) + status_info = get_mirror_statuses(mirror_id=mirror.id, + show_all=authorized) data = status_info.copy() data['version'] = 3 to_json = json.dumps(data, ensure_ascii=False, -- cgit v1.2.3-2-g168b From d83e0842053b3a4a4dfb31f373d34ecf1e0c5ce2 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 14 Dec 2013 11:15:42 -0600 Subject: Add Flyspray Bug field to mirror model Signed-off-by: Dan McGee --- mirrors/admin.py | 3 +- .../migrations/0027_auto__add_field_mirror_bug.py | 83 ++++++++++++++++++++++ mirrors/models.py | 1 + 3 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 mirrors/migrations/0027_auto__add_field_mirror_bug.py (limited to 'mirrors') diff --git a/mirrors/admin.py b/mirrors/admin.py index e35d9ce7..17365486 100644 --- a/mirrors/admin.py +++ b/mirrors/admin.py @@ -54,7 +54,7 @@ class MirrorAdminForm(forms.ModelForm): model = Mirror fields = ('name', 'tier', 'upstream', 'admin_email', 'alternate_email', 'public', 'active', 'isos', 'rsync_user', 'rsync_password', - 'notes') + 'bug', 'notes') upstream = forms.ModelChoiceField( queryset=Mirror.objects.filter(tier__gte=0, tier__lte=1), @@ -67,6 +67,7 @@ class MirrorAdmin(admin.ModelAdmin): 'isos', 'admin_email', 'alternate_email') list_filter = ('tier', 'active', 'public') search_fields = ('name', 'admin_email', 'alternate_email') + readonly_fields = ('created',) inlines = [ MirrorUrlInlineAdmin, MirrorRsyncInlineAdmin, diff --git a/mirrors/migrations/0027_auto__add_field_mirror_bug.py b/mirrors/migrations/0027_auto__add_field_mirror_bug.py new file mode 100644 index 00000000..57727333 --- /dev/null +++ b/mirrors/migrations/0027_auto__add_field_mirror_bug.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + def forwards(self, orm): + db.add_column(u'mirrors_mirror', 'bug', + self.gf('django.db.models.fields.PositiveIntegerField')(null=True), + keep_default=False) + + def backwards(self, orm): + db.delete_column(u'mirrors_mirror', 'bug') + + + models = { + u'mirrors.checklocation': { + 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'}, + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'}) + }, + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'bug': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'location': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'null': 'True', 'to': u"orm['mirrors.CheckLocation']"}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'ordering': "('ip',)", 'object_name': 'MirrorRsync'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('mirrors.fields.IPNetworkField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index da3d8c0d..47e2051b 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -28,6 +28,7 @@ class Mirror(models.Model): isos = models.BooleanField("ISOs", default=True) rsync_user = models.CharField(max_length=50, blank=True, default='') rsync_password = models.CharField(max_length=50, blank=True, default='') + bug = models.PositiveIntegerField("Flyspray bug", null=True, blank=True) notes = models.TextField(blank=True) created = models.DateTimeField(editable=False) -- cgit v1.2.3-2-g168b From a116f0d94221f72fa14d90ec77b9777efbfada65 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 14 Dec 2013 11:24:57 -0600 Subject: Add update query for extracting Flyspray bug number Signed-off-by: Dan McGee --- mirrors/migrations/0027_auto__add_field_mirror_bug.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'mirrors') diff --git a/mirrors/migrations/0027_auto__add_field_mirror_bug.py b/mirrors/migrations/0027_auto__add_field_mirror_bug.py index 57727333..f7304ba8 100644 --- a/mirrors/migrations/0027_auto__add_field_mirror_bug.py +++ b/mirrors/migrations/0027_auto__add_field_mirror_bug.py @@ -9,6 +9,14 @@ class Migration(SchemaMigration): db.add_column(u'mirrors_mirror', 'bug', self.gf('django.db.models.fields.PositiveIntegerField')(null=True), keep_default=False) + # UPDATE mirrors_mirror m + # SET bug = ( + # SELECT extracted::int FROM ( + # SELECT id, substring(notes from 'FS#([\d]+)') AS extracted FROM mirrors_mirror + # ) a + # WHERE extracted IS NOT NULL AND a.id = m.id + # ) + # WHERE notes LIKE '%FS#%'; def backwards(self, orm): db.delete_column(u'mirrors_mirror', 'bug') -- cgit v1.2.3-2-g168b From bdfa22500f47fcfa0f40de14424c25792995c9e9 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 14 Dec 2013 17:10:52 -0600 Subject: Use stable parameters for cacheable function It doesn't do much good to mark a function as cacheable if we call it every single time with different arguments due to using the current date and time. Fix it by passing the offset in instead. Signed-off-by: Dan McGee --- mirrors/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index 9f40bca6..0dd26ae0 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -22,7 +22,8 @@ def dictfetchall(cursor): ] @cache_function(178) -def status_data(cutoff_time, mirror_id=None): +def status_data(cutoff=DEFAULT_CUTOFF, mirror_id=None): + cutoff_time = now() - cutoff if mirror_id is not None: params = [cutoff_time, mirror_id] mirror_where = 'AND u.mirror_id = %s' @@ -125,7 +126,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None, show_all=False): valid_urls = valid_urls.filter(active=True, mirror__active=True, mirror__public=True) - url_data = status_data(cutoff_time, mirror_id) + url_data = status_data(cutoff, mirror_id) urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( id__in=valid_urls).order_by('mirror__id', 'url') -- cgit v1.2.3-2-g168b From 3c7b02753a4f742eeb66b8deea2fc3f179b87b8e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 15 Dec 2013 13:21:04 -0600 Subject: Add delay function to MirrorLog model Signed-off-by: Dan McGee --- mirrors/models.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'mirrors') diff --git a/mirrors/models.py b/mirrors/models.py index 47e2051b..d2c64c51 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -1,3 +1,4 @@ +from datetime import timedelta import socket from urlparse import urlparse @@ -159,6 +160,12 @@ class MirrorLog(models.Model): is_success = models.BooleanField(default=True) error = models.TextField(blank=True, default='') + def delay(self): + # sanity check, this shouldn't happen + if self.check_time < self.last_sync: + return timedelta() + return self.check_time - self.last_sync + def __unicode__(self): return "Check of %s at %s" % (self.url.url, self.check_time) -- cgit v1.2.3-2-g168b From 79aef280ddf0c704fd40d0077822a8ff7548437e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 15 Dec 2013 13:22:41 -0600 Subject: Add mirror URL details page This will allow those that care about mirrors to zoom into URL-level details for each mirror and examine the individual check results. Signed-off-by: Dan McGee --- mirrors/models.py | 2 ++ mirrors/urls.py | 1 + mirrors/views.py | 19 +++++++++++++++++++ 3 files changed, 22 insertions(+) (limited to 'mirrors') diff --git a/mirrors/models.py b/mirrors/models.py index d2c64c51..57664562 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -161,6 +161,8 @@ class MirrorLog(models.Model): error = models.TextField(blank=True, default='') def delay(self): + if self.last_sync is None: + return None # sanity check, this shouldn't happen if self.check_time < self.last_sync: return timedelta() diff --git a/mirrors/urls.py b/mirrors/urls.py index 7cf76aa1..b1054380 100644 --- a/mirrors/urls.py +++ b/mirrors/urls.py @@ -9,6 +9,7 @@ urlpatterns = patterns('mirrors.views', (r'^locations/json/$', 'locations_json', {}, 'mirror-locations-json'), (r'^(?P[\.\-\w]+)/$', 'mirror_details'), (r'^(?P[\.\-\w]+)/json/$', 'mirror_details_json'), + (r'^(?P[\.\-\w]+)/(?P\d+)/$', 'url_details'), ) # vim: set ts=4 sw=4 et: diff --git a/mirrors/views.py b/mirrors/views.py index 9e05e5fc..b2e75b25 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -186,6 +186,7 @@ def mirror_details(request, name): } return render(request, 'mirrors/mirror_details.html', context) + def mirror_details_json(request, name): authorized = request.user.is_authenticated() mirror = get_object_or_404(Mirror, name=name) @@ -199,6 +200,24 @@ def mirror_details_json(request, name): return response +def url_details(request, name, url_id): + url = get_object_or_404(MirrorUrl, id=url_id, mirror__name=name) + mirror = url.mirror + authorized = request.user.is_authenticated() + if not authorized and \ + (not mirror.public or not mirror.active or not url.active): + raise Http404 + error_cutoff = timedelta(days=7) + cutoff_time = now() - error_cutoff + logs = MirrorLog.objects.filter(url=url, check_time__gte=cutoff_time).order_by('-check_time') + + context = { + 'url': url, + 'logs': logs, + } + return render(request, 'mirrors/url_details.html', context) + + def status(request, tier=None): if tier is not None: tier = int(tier) -- cgit v1.2.3-2-g168b From 77a45dc7bc6f0badb45ec043e85f1b542c52792e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 15 Dec 2013 13:33:50 -0600 Subject: Use select_related() in new mirror URL details view Signed-off-by: Dan McGee --- mirrors/views.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'mirrors') diff --git a/mirrors/views.py b/mirrors/views.py index b2e75b25..34336165 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -201,7 +201,8 @@ def mirror_details_json(request, name): def url_details(request, name, url_id): - url = get_object_or_404(MirrorUrl, id=url_id, mirror__name=name) + url = get_object_or_404(MirrorUrl.objects.select_related(), + id=url_id, mirror__name=name) mirror = url.mirror authorized = request.user.is_authenticated() if not authorized and \ @@ -209,7 +210,8 @@ def url_details(request, name, url_id): raise Http404 error_cutoff = timedelta(days=7) cutoff_time = now() - error_cutoff - logs = MirrorLog.objects.filter(url=url, check_time__gte=cutoff_time).order_by('-check_time') + logs = MirrorLog.objects.select_related('location').filter( + url=url, check_time__gte=cutoff_time).order_by('-check_time') context = { 'url': url, -- cgit v1.2.3-2-g168b