diff options
Diffstat (limited to 'mirrors')
| -rw-r--r-- | mirrors/admin.py | 3 | ||||
| -rw-r--r-- | mirrors/migrations/0027_auto__add_field_mirror_bug.py | 91 | ||||
| -rw-r--r-- | mirrors/models.py | 10 | ||||
| -rw-r--r-- | mirrors/templatetags/mirror_status.py | 11 | ||||
| -rw-r--r-- | mirrors/urls.py | 1 | ||||
| -rw-r--r-- | mirrors/utils.py | 25 | ||||
| -rw-r--r-- | mirrors/views.py | 39 |
7 files changed, 160 insertions, 20 deletions
diff --git a/mirrors/admin.py b/mirrors/admin.py index e35d9ce7..17365486 100644 --- a/mirrors/admin.py +++ b/mirrors/admin.py @@ -54,7 +54,7 @@ class MirrorAdminForm(forms.ModelForm): model = Mirror fields = ('name', 'tier', 'upstream', 'admin_email', 'alternate_email', 'public', 'active', 'isos', 'rsync_user', 'rsync_password', - 'notes') + 'bug', 'notes') upstream = forms.ModelChoiceField( queryset=Mirror.objects.filter(tier__gte=0, tier__lte=1), @@ -67,6 +67,7 @@ class MirrorAdmin(admin.ModelAdmin): 'isos', 'admin_email', 'alternate_email') list_filter = ('tier', 'active', 'public') search_fields = ('name', 'admin_email', 'alternate_email') + readonly_fields = ('created',) inlines = [ MirrorUrlInlineAdmin, MirrorRsyncInlineAdmin, diff --git a/mirrors/migrations/0027_auto__add_field_mirror_bug.py b/mirrors/migrations/0027_auto__add_field_mirror_bug.py new file mode 100644 index 00000000..f7304ba8 --- /dev/null +++ b/mirrors/migrations/0027_auto__add_field_mirror_bug.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + def forwards(self, orm): + db.add_column(u'mirrors_mirror', 'bug', + self.gf('django.db.models.fields.PositiveIntegerField')(null=True), + keep_default=False) + # UPDATE mirrors_mirror m + # SET bug = ( + # SELECT extracted::int FROM ( + # SELECT id, substring(notes from 'FS#([\d]+)') AS extracted FROM mirrors_mirror + # ) a + # WHERE extracted IS NOT NULL AND a.id = m.id + # ) + # WHERE notes LIKE '%FS#%'; + + def backwards(self, orm): + db.delete_column(u'mirrors_mirror', 'bug') + + + models = { + u'mirrors.checklocation': { + 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'}, + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'}) + }, + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'bug': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'location': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'null': 'True', 'to': u"orm['mirrors.CheckLocation']"}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'ordering': "('ip',)", 'object_name': 'MirrorRsync'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('mirrors.fields.IPNetworkField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index da3d8c0d..57664562 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -1,3 +1,4 @@ +from datetime import timedelta import socket from urlparse import urlparse @@ -28,6 +29,7 @@ class Mirror(models.Model): isos = models.BooleanField("ISOs", default=True) rsync_user = models.CharField(max_length=50, blank=True, default='') rsync_password = models.CharField(max_length=50, blank=True, default='') + bug = models.PositiveIntegerField("Flyspray bug", null=True, blank=True) notes = models.TextField(blank=True) created = models.DateTimeField(editable=False) @@ -158,6 +160,14 @@ class MirrorLog(models.Model): is_success = models.BooleanField(default=True) error = models.TextField(blank=True, default='') + def delay(self): + if self.last_sync is None: + return None + # sanity check, this shouldn't happen + if self.check_time < self.last_sync: + return timedelta() + return self.check_time - self.last_sync + def __unicode__(self): return "Check of %s at %s" % (self.url.url, self.check_time) diff --git a/mirrors/templatetags/mirror_status.py b/mirrors/templatetags/mirror_status.py index 9a363fbe..b3810d9a 100644 --- a/mirrors/templatetags/mirror_status.py +++ b/mirrors/templatetags/mirror_status.py @@ -1,6 +1,5 @@ from datetime import timedelta from django import template -from django.template.defaultfilters import floatformat register = template.Library() @@ -27,10 +26,16 @@ def hours(value): return '%d hours' % hrs @register.filter -def percentage(value, arg=-1): +def floatvalue(value, arg=2): + if value is None: + return u'' + return '%.*f' % (arg, value) + +@register.filter +def percentage(value, arg=1): if not value and type(value) != float: return u'' new_val = value * 100.0 - return floatformat(new_val, arg) + '%' + return '%.*f%%' % (arg, new_val) # vim: set ts=4 sw=4 et: diff --git a/mirrors/urls.py b/mirrors/urls.py index 7cf76aa1..b1054380 100644 --- a/mirrors/urls.py +++ b/mirrors/urls.py @@ -9,6 +9,7 @@ urlpatterns = patterns('mirrors.views', (r'^locations/json/$', 'locations_json', {}, 'mirror-locations-json'), (r'^(?P<name>[\.\-\w]+)/$', 'mirror_details'), (r'^(?P<name>[\.\-\w]+)/json/$', 'mirror_details_json'), + (r'^(?P<name>[\.\-\w]+)/(?P<url_id>\d+)/$', 'url_details'), ) # vim: set ts=4 sw=4 et: diff --git a/mirrors/utils.py b/mirrors/utils.py index 633731a7..0dd26ae0 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -22,7 +22,8 @@ def dictfetchall(cursor): ] @cache_function(178) -def status_data(cutoff_time, mirror_id=None): +def status_data(cutoff=DEFAULT_CUTOFF, mirror_id=None): + cutoff_time = now() - cutoff if mirror_id is not None: params = [cutoff_time, mirror_id] mirror_where = 'AND u.mirror_id = %s' @@ -113,20 +114,19 @@ def annotate_url(url, url_data): url.score = (hours + url.duration_avg + stddev) / divisor -def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): +def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None, show_all=False): cutoff_time = now() - cutoff - # TODO: this prevents grabbing data points from any mirror that was active, - # receiving checks, and then marked private. we can probably be smarter and - # filter the data later? - valid_urls = MirrorUrl.objects.filter(active=True, - mirror__active=True, mirror__public=True, + valid_urls = MirrorUrl.objects.filter( logs__check_time__gte=cutoff_time).distinct() if mirror_id: valid_urls = valid_urls.filter(mirror_id=mirror_id) + if not show_all: + valid_urls = valid_urls.filter(active=True, mirror__active=True, + mirror__public=True) - url_data = status_data(cutoff_time, mirror_id) + url_data = status_data(cutoff, mirror_id) urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( id__in=valid_urls).order_by('mirror__id', 'url') @@ -159,11 +159,11 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): } -def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None): +def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None, show_all=False): cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( - is_success=False, check_time__gte=cutoff_time, url__active=True, - url__mirror__active=True, url__mirror__public=True).values( + is_success=False, check_time__gte=cutoff_time, + url__mirror__public=True).values( 'url__url', 'url__country', 'url__protocol__protocol', 'url__mirror__tier', 'error').annotate( error_count=Count('error'), last_occurred=Max('check_time') @@ -171,6 +171,9 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None): if mirror_id: errors = errors.filter(url__mirror_id=mirror_id) + if not show_all: + errors = errors.filter(url__active=True, url__mirror__active=True, + url__mirror__public=True) errors = list(errors) for err in errors: diff --git a/mirrors/views.py b/mirrors/views.py index ec056696..34336165 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -153,15 +153,20 @@ def mirrors(request): def mirror_details(request, name): mirror = get_object_or_404(Mirror, name=name) - if not request.user.is_authenticated() and \ + authorized = request.user.is_authenticated() + if not authorized and \ (not mirror.public or not mirror.active): raise Http404 error_cutoff = timedelta(days=7) - status_info = get_mirror_statuses(mirror_id=mirror.id) + status_info = get_mirror_statuses(mirror_id=mirror.id, + show_all=authorized) checked_urls = {url for url in status_info['urls'] \ if url.mirror_id == mirror.id} - all_urls = set(mirror.urls.filter(active=True).select_related('protocol')) + all_urls = mirror.urls.select_related('protocol') + if not authorized: + all_urls = all_urls.filter(active=True) + all_urls = set(all_urls) # Add dummy data for URLs that we haven't checked recently other_urls = all_urls.difference(checked_urls) for url in other_urls: @@ -170,7 +175,8 @@ def mirror_details(request, name): setattr(url, attr, None) all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url')) - error_logs = get_mirror_errors(mirror_id=mirror.id, cutoff=error_cutoff) + error_logs = get_mirror_errors(mirror_id=mirror.id, cutoff=error_cutoff, + show_all=True) context = { 'mirror': mirror, @@ -180,9 +186,12 @@ def mirror_details(request, name): } return render(request, 'mirrors/mirror_details.html', context) + def mirror_details_json(request, name): + authorized = request.user.is_authenticated() mirror = get_object_or_404(Mirror, name=name) - status_info = get_mirror_statuses(mirror_id=mirror.id) + status_info = get_mirror_statuses(mirror_id=mirror.id, + show_all=authorized) data = status_info.copy() data['version'] = 3 to_json = json.dumps(data, ensure_ascii=False, @@ -191,6 +200,26 @@ def mirror_details_json(request, name): return response +def url_details(request, name, url_id): + url = get_object_or_404(MirrorUrl.objects.select_related(), + id=url_id, mirror__name=name) + mirror = url.mirror + authorized = request.user.is_authenticated() + if not authorized and \ + (not mirror.public or not mirror.active or not url.active): + raise Http404 + error_cutoff = timedelta(days=7) + cutoff_time = now() - error_cutoff + logs = MirrorLog.objects.select_related('location').filter( + url=url, check_time__gte=cutoff_time).order_by('-check_time') + + context = { + 'url': url, + 'logs': logs, + } + return render(request, 'mirrors/url_details.html', context) + + def status(request, tier=None): if tier is not None: tier = int(tier) |