diff options
Diffstat (limited to 'mirrors')
| -rw-r--r-- | mirrors/admin.py | 14 | ||||
| -rw-r--r-- | mirrors/fields.py | 49 | ||||
| -rw-r--r-- | mirrors/management/commands/mirrorcheck.py | 12 | ||||
| -rw-r--r-- | mirrors/migrations/0025_auto__chg_field_mirrorrsync_ip.py | 85 | ||||
| -rw-r--r-- | mirrors/models.py | 14 | ||||
| -rw-r--r-- | mirrors/static/mirror_status.js | 76 | ||||
| -rw-r--r-- | mirrors/urls.py | 1 | ||||
| -rw-r--r-- | mirrors/utils.py | 163 | ||||
| -rw-r--r-- | mirrors/views.py | 44 |
9 files changed, 346 insertions, 112 deletions
diff --git a/mirrors/admin.py b/mirrors/admin.py index d6ea3950..9c88207d 100644 --- a/mirrors/admin.py +++ b/mirrors/admin.py @@ -1,4 +1,3 @@ -import re from urlparse import urlparse, urlunsplit from django import forms @@ -36,22 +35,9 @@ class MirrorUrlInlineAdmin(admin.TabularInline): extra = 3 -# ripped off from django.forms.fields, adding netmask ability -IPV4NM_RE = re.compile(r'^(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}(/(\d|[1-2]\d|3[0-2])){0,1}$') - -class IPAddressNetmaskField(forms.fields.RegexField): - default_error_messages = { - 'invalid': u'Enter a valid IPv4 address, possibly including netmask.', - } - - def __init__(self, *args, **kwargs): - super(IPAddressNetmaskField, self).__init__(IPV4NM_RE, *args, **kwargs) - - class MirrorRsyncForm(forms.ModelForm): class Meta: model = MirrorRsync - ip = IPAddressNetmaskField(label='IP') class MirrorRsyncInlineAdmin(admin.TabularInline): diff --git a/mirrors/fields.py b/mirrors/fields.py new file mode 100644 index 00000000..206c9d7d --- /dev/null +++ b/mirrors/fields.py @@ -0,0 +1,49 @@ +from IPy import IP + +from django import forms +from django.core import validators +from django.core.exceptions import ValidationError +from django.db import models +from south.modelsinspector import add_introspection_rules + + +class IPNetworkFormField(forms.Field): + def to_python(self, value): + if value in validators.EMPTY_VALUES: + return None + try: + value = IP(value) + except ValueError as e: + raise ValidationError(str(e)) + return value + + +class IPNetworkField(models.Field): + __metaclass__ = models.SubfieldBase + description = "IPv4 or IPv6 address or subnet" + + def __init__(self, *args, **kwargs): + kwargs['max_length'] = 44 + super(IPNetworkField, self).__init__(*args, **kwargs) + + def get_internal_type(self): + return "IPAddressField" + + def to_python(self, value): + if not value: + return None + return IP(value) + + def get_prep_value(self, value): + value = self.to_python(value) + if not value: + return None + return str(value) + + def formfield(self, **kwargs): + defaults = {'form_class': IPNetworkFormField} + defaults.update(kwargs) + return super(IPNetworkField, self).formfield(**defaults) + + +add_introspection_rules([], ["^mirrors\.fields\.IPNetworkField"]) diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index d6de8f22..e7dd7b49 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -106,19 +106,13 @@ def parse_lastsync(log, data): def check_mirror_url(mirror_url, location, timeout): - if location: - if location.family == socket.AF_INET6: - ipopt = '--ipv6' - elif location.family == socket.AF_INET: - ipopt = '--ipv4' - url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) log = MirrorLog(url=mirror_url, check_time=now(), location=location) headers = {'User-Agent': 'archweb/1.0'} req = urllib2.Request(url, None, headers) + start = time.time() try: - start = time.time() result = urllib2.urlopen(req, timeout=timeout) data = result.read() result.close() @@ -147,12 +141,12 @@ def check_mirror_url(mirror_url, location, timeout): elif isinstance(e.reason, socket.error): log.error = e.reason.args[1] logger.debug("failed: %s, %s", url, log.error) - except HTTPException as e: + except HTTPException: # e.g., BadStatusLine log.is_success = False log.error = "Exception in processing HTTP request." logger.debug("failed: %s, %s", url, log.error) - except socket.timeout as e: + except socket.timeout: log.is_success = False log.error = "Connection timed out." logger.debug("failed: %s, %s", url, log.error) diff --git a/mirrors/migrations/0025_auto__chg_field_mirrorrsync_ip.py b/mirrors/migrations/0025_auto__chg_field_mirrorrsync_ip.py new file mode 100644 index 00000000..b359b637 --- /dev/null +++ b/mirrors/migrations/0025_auto__chg_field_mirrorrsync_ip.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + if db.backend_name == 'postgres': + # For PostgreSQL, because it uses the 'inet' type and not a varchar + # column, we need to add an explict 'USING' cast to the SQL + # statement. We then execute the alter_column as well to ensure any + # of the other side-effects happen. + db.execute('ALTER TABLE "mirrors_mirrorrsync" ALTER COLUMN "ip" TYPE inet USING "ip"::inet') + db.alter_column(u'mirrors_mirrorrsync', 'ip', self.gf('mirrors.fields.IPNetworkField')(max_length=44)) + + def backwards(self, orm): + db.alter_column(u'mirrors_mirrorrsync', 'ip', self.gf('django.db.models.fields.CharField')(max_length=44)) + + models = { + u'mirrors.checklocation': { + 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'}, + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'}) + }, + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'location': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'null': 'True', 'to': u"orm['mirrors.CheckLocation']"}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('mirrors.fields.IPNetworkField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index e41f6b22..d8ac7952 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -6,6 +6,7 @@ from django.db import models from django.db.models.signals import pre_save from django_countries import CountryField +from .fields import IPNetworkField from main.utils import set_created_field @@ -91,7 +92,7 @@ class MirrorUrl(models.Model): families = self.address_families() self.has_ipv4 = socket.AF_INET in families self.has_ipv6 = socket.AF_INET6 in families - except socket.error as e: + except socket.error: # We don't fail in this case; we'll just set both to False self.has_ipv4 = False self.has_ipv6 = False @@ -105,7 +106,7 @@ class MirrorUrl(models.Model): class MirrorRsync(models.Model): # max length is 40 chars for full-form IPv6 addr + subnet - ip = models.CharField("IP", max_length=44) + ip = IPNetworkField("IP") mirror = models.ForeignKey(Mirror, related_name="rsync_ips") created = models.DateTimeField(editable=False) @@ -136,6 +137,15 @@ class CheckLocation(models.Model): families = [x[0] for x in info] return families[0] + @property + def ip_version(self): + '''Returns integer '4' or '6'.''' + if self.family == socket.AF_INET6: + return 6 + if self.family == socket.AF_INET: + return 4 + return None + class MirrorLog(models.Model): url = models.ForeignKey(MirrorUrl, related_name="logs") diff --git a/mirrors/static/mirror_status.js b/mirrors/static/mirror_status.js index 8ec85c40..241f5c61 100644 --- a/mirrors/static/mirror_status.js +++ b/mirrors/static/mirror_status.js @@ -1,13 +1,23 @@ -function mirror_status(chart_id, data_url) { - var jq_div = jQuery(chart_id); +function draw_graphs(location_url, log_url, container_id) { + jQuery.when(jQuery.getJSON(location_url), jQuery.getJSON(log_url)) + .then(function(loc_data, log_data) { + /* use the same color selection for a given URL in every graph */ + var color = d3.scale.category10(); + jQuery.each(loc_data[0].locations, function(i, val) { + mirror_status(container_id, val, log_data[0], color); + }); + }); +} - var draw_graph = function(data) { +function mirror_status(container_id, check_loc, log_data, color) { + + var draw_graph = function(chart_id, data) { + var jq_div = jQuery(chart_id); var margin = {top: 20, right: 20, bottom: 30, left: 40}, width = jq_div.width() - margin.left - margin.right, height = jq_div.height() - margin.top - margin.bottom; - var color = d3.scale.category10(), - x = d3.time.scale.utc().range([0, width]), + var x = d3.time.scale.utc().range([0, width]), y = d3.scale.linear().range([height, 0]), x_axis = d3.svg.axis().scale(x).orient("bottom"), y_axis = d3.svg.axis().scale(y).orient("left"); @@ -86,8 +96,9 @@ function mirror_status(chart_id, data_url) { .text(function(d) { return d.url + "\n" + d.duration.toFixed(3) + " secs\n" + d.check_time.toUTCString(); }); /* add a legend for good measure */ + var active = jQuery.map(data, function(item, i) { return item.url; }); var legend = svg.selectAll(".legend") - .data(color.domain()) + .data(active) .enter().append("g") .attr("class", "legend") .attr("transform", function(d, i) { return "translate(0," + i * 20 + ")"; }); @@ -106,31 +117,52 @@ function mirror_status(chart_id, data_url) { .text(function(d) { return d; }); }; - /* invoke the data-fetch + first draw */ - var cached_data = null; - d3.json(data_url, function(json) { - cached_data = jQuery.map(json.urls, function(url, i) { + var filter_data = function(json, location_id) { + return jQuery.map(json.urls, function(url, i) { + var logs = jQuery.map(url.logs, function(log, j) { + if (!log.is_success) { + return null; + } + /* screen by location ID if we were given one */ + if (location_id && log.location_id !== location_id) { + return null; + } + return { + duration: log.duration, + check_time: new Date(log.check_time) + }; + }); + /* don't return URLs without any log info */ + if (logs.length === 0) { + return null; + } return { url: url.url, - logs: jQuery.map(url.logs, function(log, j) { - if (!log.is_success) { - return null; - } - return { - duration: log.duration, - check_time: new Date(log.check_time) - }; - }) + logs: logs }; }); - draw_graph(cached_data); - }); + }; + + var cached_data = filter_data(log_data, check_loc.id); + /* we had a check location with no log data handed to us, skip graphing */ + if (cached_data.length === 0) { + return; + } + + /* create the containers, defer the actual graph drawing */ + var chart_id = 'status-chart-' + check_loc.id; + jQuery(container_id).append('<h3><span class="fam-flag fam-flag-' + check_loc.country_code.toLowerCase() + '" title="' + check_loc.country + '"></span> ' + check_loc.country + ' (' + check_loc.source_ip + '), IPv' + check_loc.ip_version + '</h3>'); + jQuery(container_id).append('<div id="' + chart_id + '" class="visualize-mirror visualize-chart"></div>'); + jQuery(container_id).append('<br/>'); + setTimeout(function() { + draw_graph('#' + chart_id, cached_data); + }, 0); /* then hook up a resize handler to redraw if necessary */ var resize_timeout = null; var real_resize = function() { resize_timeout = null; - draw_graph(cached_data); + draw_graph('#' + chart_id, cached_data); }; jQuery(window).resize(function() { if (resize_timeout) { diff --git a/mirrors/urls.py b/mirrors/urls.py index 4e929410..7cf76aa1 100644 --- a/mirrors/urls.py +++ b/mirrors/urls.py @@ -6,6 +6,7 @@ urlpatterns = patterns('mirrors.views', (r'^status/json/$', 'status_json', {}, 'mirror-status-json'), (r'^status/tier/(?P<tier>\d+)/$', 'status', {}, 'mirror-status-tier'), (r'^status/tier/(?P<tier>\d+)/json/$', 'status_json', {}, 'mirror-status-tier-json'), + (r'^locations/json/$', 'locations_json', {}, 'mirror-locations-json'), (r'^(?P<name>[\.\-\w]+)/$', 'mirror_details'), (r'^(?P<name>[\.\-\w]+)/json/$', 'mirror_details_json'), ) diff --git a/mirrors/utils.py b/mirrors/utils.py index 3ab176b3..ba45da5f 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -1,22 +1,107 @@ from datetime import timedelta -from django.db.models import Avg, Count, Max, Min, StdDev +from django.db import connection +from django.db.models import Count, Max, Min +from django.utils.dateparse import parse_datetime from django.utils.timezone import now from django_countries.fields import Country from main.utils import cache_function, database_vendor -from .models import MirrorLog, MirrorProtocol, MirrorUrl +from .models import MirrorLog, MirrorUrl DEFAULT_CUTOFF = timedelta(hours=24) -def annotate_url(url, delays): + +def dictfetchall(cursor): + "Returns all rows from a cursor as a dict." + desc = cursor.description + return [ + dict(zip([col[0] for col in desc], row)) + for row in cursor.fetchall() + ] + + +def status_data(cutoff_time, mirror_id=None): + if mirror_id is not None: + params = [cutoff_time, mirror_id] + mirror_where = 'AND u.mirror_id = %s' + else: + params = [cutoff_time] + mirror_where = '' + + vendor = database_vendor(MirrorUrl) + if vendor == 'sqlite': + sql = """ +SELECT l.url_id, u.mirror_id, + COUNT(l.id) AS check_count, + COUNT(l.duration) AS success_count, + MAX(l.last_sync) AS last_sync, + MAX(l.check_time) AS last_check, + AVG(l.duration) AS duration_avg, + 0.0 AS duration_stddev, + AVG(STRFTIME('%%s', check_time) - STRFTIME('%%s', last_sync)) AS delay +FROM mirrors_mirrorlog l +JOIN mirrors_mirrorurl u ON u.id = l.url_id +WHERE l.check_time >= %s +""" + mirror_where + """ +GROUP BY l.url_id, u.mirror_id +""" + else: + sql = """ +SELECT l.url_id, u.mirror_id, + COUNT(l.id) AS check_count, + COUNT(l.duration) AS success_count, + MAX(l.last_sync) AS last_sync, + MAX(l.check_time) AS last_check, + AVG(l.duration) AS duration_avg, + STDDEV(l.duration) AS duration_stddev, + AVG(check_time - last_sync) AS delay +FROM mirrors_mirrorlog l +JOIN mirrors_mirrorurl u ON u.id = l.url_id +WHERE l.check_time >= %s +""" + mirror_where + """ +GROUP BY l.url_id, u.mirror_id +""" + + cursor = connection.cursor() + cursor.execute(sql, params) + url_data = dictfetchall(cursor) + + # sqlite loves to return less than ideal types + if vendor == 'sqlite': + for item in url_data: + if item['delay'] is not None: + item['delay'] = timedelta(seconds=item['delay']) + if item['last_sync'] is not None: + item['last_sync'] = parse_datetime(item['last_sync']) + item['last_check'] = parse_datetime(item['last_check']) + + return {item['url_id']: item for item in url_data} + + +def annotate_url(url, url_data): '''Given a MirrorURL object, add a few more attributes to it regarding status, including completion_pct, delay, and score.''' - url.completion_pct = float(url.success_count) / url.check_count - if url.id in delays: - url_delays = delays[url.id] - url.delay = sum(url_delays, timedelta()) / len(url_delays) + known_attrs = ( + ('success_count', 0), + ('check_count', 0), + ('completion_pct', None), + ('last_check', None), + ('last_sync', None), + ('delay', None), + ('score', None), + ) + for k, v in known_attrs: + setattr(url, k, v) + for k, v in url_data.items(): + if k not in ('url_id', 'mirror_id'): + setattr(url, k, v) + + if url.check_count > 0: + url.completion_pct = float(url.success_count) / url.check_count + + if url.delay is not None: hours = url.delay.days * 24.0 + url.delay.seconds / 3600.0 if url.completion_pct > 0: @@ -24,63 +109,32 @@ def annotate_url(url, delays): else: # arbitrary small value divisor = 0.005 - url.score = (hours + url.duration_avg + url.duration_stddev) / divisor - else: - url.delay = None - url.score = None + stddev = url.duration_stddev or 0.0 + url.score = (hours + url.duration_avg + stddev) / divisor -@cache_function(123) -def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): +def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): cutoff_time = now() - cutoff valid_urls = MirrorUrl.objects.filter( mirror__active=True, mirror__public=True, logs__check_time__gte=cutoff_time).distinct() - if mirror_ids: - valid_urls = valid_urls.filter(mirror_id__in=mirror_ids) - - url_data = MirrorUrl.objects.values('id', 'mirror_id').filter( - id__in=valid_urls, logs__check_time__gte=cutoff_time).annotate( - check_count=Count('logs'), - success_count=Count('logs__duration'), - last_sync=Max('logs__last_sync'), - last_check=Max('logs__check_time'), - duration_avg=Avg('logs__duration')) - - vendor = database_vendor(MirrorUrl) - if vendor != 'sqlite': - url_data = url_data.annotate(duration_stddev=StdDev('logs__duration')) + if mirror_id: + valid_urls = valid_urls.filter(mirror_id=mirror_id) + url_data = status_data(cutoff_time, mirror_id) urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( id__in=valid_urls).order_by('mirror__id', 'url') - # The Django ORM makes it really hard to get actual average delay in the - # above query, so run a seperate query for it and we will process the - # results here. - times = MirrorLog.objects.values_list( - 'url_id', 'check_time', 'last_sync').filter( - is_success=True, last_sync__isnull=False, - check_time__gte=cutoff_time) - if mirror_ids: - times = times.filter(url__mirror_id__in=mirror_ids) - delays = {} - for url_id, check_time, last_sync in times: - delay = check_time - last_sync - delays.setdefault(url_id, []).append(delay) - if urls: - url_data = dict((item['id'], item) for item in url_data) for url in urls: - for k, v in url_data.get(url.id, {}).items(): - if k not in ('id', 'mirror_id'): - setattr(url, k, v) - last_check = max([u.last_check for u in urls]) + annotate_url(url, url_data.get(url.id, {})) + last_check = max([u.last_check for u in urls if u.last_check]) num_checks = max([u.check_count for u in urls]) check_info = MirrorLog.objects.filter(check_time__gte=cutoff_time) - if mirror_ids: - check_info = check_info.filter(url__mirror_id__in=mirror_ids) + if mirror_id: + check_info = check_info.filter(url__mirror_id=mirror_id) check_info = check_info.aggregate( mn=Min('check_time'), mx=Max('check_time')) if num_checks > 1: @@ -93,12 +147,6 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): num_checks = 0 check_frequency = None - for url in urls: - # fake the standard deviation for local testing setups - if vendor == 'sqlite': - setattr(url, 'duration_stddev', 0.0) - annotate_url(url, delays) - return { 'cutoff': cutoff, 'last_check': last_check, @@ -108,8 +156,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): } -@cache_function(117) -def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None): +def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None): cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( is_success=False, check_time__gte=cutoff_time, @@ -119,8 +166,8 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None): error_count=Count('error'), last_occurred=Max('check_time') ).order_by('-last_occurred', '-error_count') - if mirror_ids: - urls = urls.filter(mirror_id__in=mirror_ids) + if mirror_id: + errors = errors.filter(url__mirror_id=mirror_id) errors = list(errors) for err in errors: diff --git a/mirrors/views.py b/mirrors/views.py index 56397633..73d40297 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -13,11 +13,10 @@ from django.utils.timezone import now from django.views.decorators.csrf import csrf_exempt from django_countries.countries import COUNTRIES -from .models import Mirror, MirrorUrl, MirrorProtocol, MirrorLog +from .models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog, + CheckLocation) from .utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF -COUNTRY_LOOKUP = dict(COUNTRIES) - class MirrorlistForm(forms.Form): country = forms.MultipleChoiceField(required=False) @@ -28,6 +27,8 @@ class MirrorlistForm(forms.Form): widget=CheckboxSelectMultiple) use_mirror_status = forms.BooleanField(required=False) + countries = dict(COUNTRIES) + def __init__(self, *args, **kwargs): super(MirrorlistForm, self).__init__(*args, **kwargs) fields = self.fields @@ -45,7 +46,7 @@ class MirrorlistForm(forms.Form): country_codes.update(MirrorUrl.objects.filter( mirror__active=True).exclude(country='').values_list( 'country', flat=True).order_by().distinct()) - countries = [(code, COUNTRY_LOOKUP[code]) for code in country_codes] + countries = [(code, self.countries[code]) for code in country_codes] return sorted(countries, key=itemgetter(1)) def as_div(self): @@ -175,7 +176,7 @@ def mirror_details(request, name): (not mirror.public or not mirror.active): raise Http404 - status_info = get_mirror_statuses(mirror_ids=[mirror.id]) + status_info = get_mirror_statuses(mirror_id=mirror.id) checked_urls = {url for url in status_info['urls'] \ if url.mirror_id == mirror.id} all_urls = set(mirror.urls.select_related('protocol')) @@ -193,7 +194,7 @@ def mirror_details(request, name): def mirror_details_json(request, name): mirror = get_object_or_404(Mirror, name=name) - status_info = get_mirror_statuses(mirror_ids=[mirror.id]) + status_info = get_mirror_statuses(mirror_id=mirror.id) data = status_info.copy() data['version'] = 3 to_json = json.dumps(data, ensure_ascii=False, @@ -264,7 +265,8 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder): class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): '''Adds URL check history information.''' - log_attributes = ('check_time', 'last_sync', 'duration', 'is_success') + log_attributes = ('check_time', 'last_sync', 'duration', 'is_success', + 'location_id') def default(self, obj): if isinstance(obj, MirrorUrl): @@ -292,4 +294,32 @@ def status_json(request, tier=None): response = HttpResponse(to_json, content_type='application/json') return response + +class LocationJSONEncoder(DjangoJSONEncoder): + '''Base JSONEncoder extended to handle CheckLocation objects.''' + + def default(self, obj): + if hasattr(obj, '__iter__'): + # mainly for queryset serialization + return list(obj) + if isinstance(obj, CheckLocation): + return { + 'id': obj.pk, + 'hostname': obj.hostname, + 'source_ip': obj.source_ip, + 'country': unicode(obj.country.name), + 'country_code': obj.country.code, + 'ip_version': obj.ip_version, + } + return super(LocationJSONEncoder, self).default(obj) + + +def locations_json(request): + data = {} + data['version'] = 1 + data['locations'] = CheckLocation.objects.all().order_by('pk') + to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + # vim: set ts=4 sw=4 et: |