From 8d21e9f4b1d5a31880f9973d758de9becc90eb39 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 28 Apr 2012 18:25:37 -0500 Subject: mirrorresolv: only run update query if values changed 98% of the time, we won't need to update the existing values as it will be the same as the prior run of this command. Do a quick check of the old and new values and don't send anything to the database if there is no need for an update. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorresolv.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorresolv.py b/mirrors/management/commands/mirrorresolv.py index 4e812f2d..0370f8ed 100644 --- a/mirrors/management/commands/mirrorresolv.py +++ b/mirrors/management/commands/mirrorresolv.py @@ -41,13 +41,19 @@ def resolve_mirrors(): logger.debug("requesting list of mirror URLs") for mirrorurl in MirrorUrl.objects.filter(mirror__active=True): try: + # save old values, we can skip no-op updates this way + oldvals = (mirrorurl.has_ipv4, mirrorurl.has_ipv6) logger.debug("resolving %3i (%s)", mirrorurl.id, mirrorurl.hostname) families = mirrorurl.address_families() mirrorurl.has_ipv4 = socket.AF_INET in families mirrorurl.has_ipv6 = socket.AF_INET6 in families logger.debug("%s: v4: %s v6: %s", mirrorurl.hostname, mirrorurl.has_ipv4, mirrorurl.has_ipv6) - mirrorurl.save(force_update=True) + # now check new values, only update if new != old + newvals = (mirrorurl.has_ipv4, mirrorurl.has_ipv6) + if newvals != oldvals: + logger.debug("values changed for %s", mirrorurl) + mirrorurl.save(force_update=True) except socket.error, e: logger.warn("error resolving %s: %s", mirrorurl.hostname, e) -- cgit v1.2.3-2-g168b From 44eb2d5ee0fa9e1b495027cec3e663ff85c0ed1d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 29 Apr 2012 21:26:23 -0500 Subject: Use a custom User-Agent when checking mirror URLs Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 7ffb7773..c1269226 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -35,6 +35,7 @@ logging.basicConfig( stream=sys.stderr) logger = logging.getLogger() + class Command(NoArgsCommand): help = "Runs a check on all known mirror URLs to determine their up-to-date status." @@ -49,13 +50,16 @@ class Command(NoArgsCommand): return check_current_mirrors() + def check_mirror_url(mirror_url): url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) log = MirrorLog(url=mirror_url, check_time=utc_now()) + headers = {'User-Agent': 'archweb/1.0'} + req = urllib2.Request(url, None, headers) try: start = time.time() - result = urllib2.urlopen(url, timeout=10) + result = urllib2.urlopen(req, timeout=10) data = result.read() result.close() end = time.time() @@ -104,6 +108,7 @@ def check_mirror_url(mirror_url): return log + def mirror_url_worker(work, output): while True: try: @@ -116,11 +121,12 @@ def mirror_url_worker(work, output): except Empty: return 0 + class MirrorCheckPool(object): - def __init__(self, work, num_threads=10): + def __init__(self, urls, num_threads=10): self.tasks = Queue() self.logs = deque() - for i in list(work): + for i in list(urls): self.tasks.put(i) self.threads = [] for i in range(num_threads): @@ -140,6 +146,7 @@ class MirrorCheckPool(object): MirrorLog.objects.bulk_create(self.logs) logger.debug("log entries saved") + def check_current_mirrors(): urls = MirrorUrl.objects.filter( protocol__is_download=True, @@ -149,8 +156,4 @@ def check_current_mirrors(): pool.run() return 0 -# For lack of a better place to put it, here is a query to get latest check -# result joined with mirror details: -# SELECT mu.*, m.*, ml.* FROM mirrors_mirrorurl mu JOIN mirrors_mirror m ON mu.mirror_id = m.id JOIN mirrors_mirrorlog ml ON mu.id = ml.url_id LEFT JOIN mirrors_mirrorlog ml2 ON ml.url_id = ml2.url_id AND ml.id < ml2.id WHERE ml2.id IS NULL AND m.active = 1 AND m.public = 1; - # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From d2d0895f13835569ff25a3161ddb94cd655dfd4f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 2 May 2012 12:23:21 -0500 Subject: Allow mirrorlist generator pattern to match any protocol Add a helper method that checks if we know about the protocol; if so, we can spit out a URL for it. This allows (if you are insane) generation of an rsync mirrorlist, for instance. Signed-off-by: Dan McGee --- mirrors/urls_mirrorlist.py | 5 +---- mirrors/views.py | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 9 deletions(-) (limited to 'mirrors') diff --git a/mirrors/urls_mirrorlist.py b/mirrors/urls_mirrorlist.py index e0f44c78..1444eca9 100644 --- a/mirrors/urls_mirrorlist.py +++ b/mirrors/urls_mirrorlist.py @@ -3,10 +3,7 @@ from django.conf.urls import patterns urlpatterns = patterns('mirrors.views', (r'^$', 'generate_mirrorlist', {}, 'mirrorlist'), (r'^all/$', 'find_mirrors', {'countries': ['all']}), - (r'^all/ftp/$', 'find_mirrors', - {'countries': ['all'], 'protocols': ['ftp']}), - (r'^all/http/$', 'find_mirrors', - {'countries': ['all'], 'protocols': ['http']}), + (r'^all/(?P[A-z]+)/$', 'find_mirrors_simple') ) # vim: set ts=4 sw=4 et: diff --git a/mirrors/views.py b/mirrors/views.py index c52656f7..6f37ace1 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -82,12 +82,15 @@ def generate_mirrorlist(request): def find_mirrors(request, countries=None, protocols=None, use_status=False, ipv4_supported=True, ipv6_supported=True): if not protocols: - protocols = MirrorProtocol.objects.filter( - is_download=True).values_list('protocol', flat=True) + protocols = MirrorProtocol.objects.filter(is_download=True) + elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol: + # we already have a queryset, no need to query again + pass + else: + protocols = MirrorProtocol.objects.filter(protocol__in=protocols) qset = MirrorUrl.objects.select_related().filter( - protocol__protocol__in=protocols, - mirror__public=True, mirror__active=True, - ) + protocol__in=protocols, + mirror__public=True, mirror__active=True) if countries and 'all' not in countries: qset = qset.filter(Q(country__in=countries) | Q(mirror__country__in=countries)) @@ -124,6 +127,11 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, mimetype='text/plain') +def find_mirrors_simple(request, protocol): + proto = get_object_or_404(MirrorProtocol, protocol=protocol) + return find_mirrors(request, protocols=[proto]) + + def mirrors(request): mirror_list = Mirror.objects.select_related().order_by('tier', 'country') if not request.user.is_authenticated(): -- cgit v1.2.3-2-g168b From 12bf4c1b1e7df2d934b9dfde8629137dedeea99f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 18 Feb 2012 19:17:00 -0600 Subject: Add a smart protocol filter to mirrorlist generator This will only list FTP mirrors for a given country if there are no HTTP mirrors available, since FTP must die. Signed-off-by: Dan McGee --- mirrors/views.py | 55 ++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 15 deletions(-) (limited to 'mirrors') diff --git a/mirrors/views.py b/mirrors/views.py index 6f37ace1..eac78ff2 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -1,4 +1,5 @@ from datetime import timedelta +from itertools import groupby from operator import attrgetter, itemgetter from django import forms @@ -79,8 +80,34 @@ def generate_mirrorlist(request): {'mirrorlist_form': form}) +def default_protocol_filter(original_urls): + key_func = attrgetter('real_country') + sorted_urls = sorted(original_urls, key=key_func) + urls = [] + for _, group in groupby(sorted_urls, key=key_func): + cntry_urls = list(group) + if any(url.protocol.default for url in cntry_urls): + cntry_urls = [url for url in cntry_urls if url.protocol.default] + urls.extend(cntry_urls) + return urls + + +def status_filter(original_urls): + status_info = get_mirror_statuses() + scores = dict((u.id, u.score) for u in status_info['urls']) + urls = [] + for u in original_urls: + u.score = scores.get(u.id, None) + # also include mirrors that don't have an up to date score + # (as opposed to those that have been set with no score) + if (u.id not in scores) or (u.score and u.score < 100.0): + urls.append(u) + # if a url doesn't have a score, treat it as the highest possible + return sorted(urls, key=lambda x: x.score or 100.0) + + def find_mirrors(request, countries=None, protocols=None, use_status=False, - ipv4_supported=True, ipv6_supported=True): + ipv4_supported=True, ipv6_supported=True, smart_protocol=False): if not protocols: protocols = MirrorProtocol.objects.filter(is_download=True) elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol: @@ -102,23 +129,17 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, ip_version |= Q(has_ipv6=True) qset = qset.filter(ip_version) + if smart_protocol: + urls = default_protocol_filter(qset) + else: + urls = qset + if not use_status: - urls = qset.order_by('mirror__name', 'url') - urls = sorted(urls, key=attrgetter('real_country')) + sort_key = attrgetter('real_country', 'mirror.name', 'url') + urls = sorted(urls, key=sort_key) template = 'mirrors/mirrorlist.txt' else: - status_info = get_mirror_statuses() - scores = dict([(u.id, u.score) for u in status_info['urls']]) - urls = [] - for u in qset: - u.score = scores.get(u.id, None) - # also include mirrors that don't have an up to date score - # (as opposed to those that have been set with no score) - if (u.id not in scores) or \ - (u.score and u.score < 100.0): - urls.append(u) - # if a url doesn't have a score, treat it as the highest possible - urls = sorted(urls, key=lambda x: x.score or 100.0) + urls = status_filter(urls) template = 'mirrors/mirrorlist_status.txt' return direct_to_template(request, template, { @@ -128,6 +149,10 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, def find_mirrors_simple(request, protocol): + if protocol == 'smart': + # generate a 'smart' mirrorlist, one that only includes FTP mirrors if + # no HTTP mirror is available in that country. + return find_mirrors(request, smart_protocol=True) proto = get_object_or_404(MirrorProtocol, protocol=protocol) return find_mirrors(request, protocols=[proto]) -- cgit v1.2.3-2-g168b From a5f5557493446bede78adb0584c88208234f874e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 12 May 2012 09:32:30 -0500 Subject: Use python json module directly in place of simplejson As of Python 2.6, this is a builtin module that has all the same functions and capabilities of the Django simplejson module. Additionally simplejson is deprecated in the upcoming Django 1.5 release. Signed-off-by: Dan McGee --- mirrors/views.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'mirrors') diff --git a/mirrors/views.py b/mirrors/views.py index eac78ff2..b0be6238 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -1,5 +1,6 @@ from datetime import timedelta from itertools import groupby +import json from operator import attrgetter, itemgetter from django import forms @@ -10,7 +11,6 @@ from django.http import Http404, HttpResponse from django.shortcuts import get_object_or_404 from django.views.decorators.csrf import csrf_exempt from django.views.generic.simple import direct_to_template -from django.utils import simplejson from django_countries.countries import COUNTRIES from .models import Mirror, MirrorUrl, MirrorProtocol @@ -237,8 +237,7 @@ def status_json(request): status_info = get_mirror_statuses() data = status_info.copy() data['version'] = 3 - to_json = simplejson.dumps(data, ensure_ascii=False, - cls=MirrorStatusJSONEncoder) + to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder) response = HttpResponse(to_json, mimetype='application/json') return response -- cgit v1.2.3-2-g168b From 76516cae45f3d1080065608bdb8f2d086322012f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 13 May 2012 19:45:57 -0500 Subject: Don't limit protocols returned by mirror status function If results weren't available for certain URLs, they won't show up anyway in this list, and if we start to check rsync URLs, then we want their values to come back in this status list. Signed-off-by: Dan McGee --- mirrors/utils.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index 32fa3587..54de567e 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -32,11 +32,9 @@ def annotate_url(url, delays): @cache_function(123) def get_mirror_statuses(cutoff=default_cutoff): cutoff_time = utc_now() - cutoff - protocols = list(MirrorProtocol.objects.filter(is_download=True)) # I swear, this actually has decent performance... urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( mirror__active=True, mirror__public=True, - protocol__in=protocols, logs__check_time__gte=cutoff_time).annotate( check_count=Count('logs'), success_count=Count('logs__duration'), -- cgit v1.2.3-2-g168b From a570c2c7fa404c2eb1760c6fb428047083c0c957 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 13 May 2012 20:10:46 -0500 Subject: Change mirror log error text to unlimited length Use TextField rather than a limited-length CharField; leave it up to the code filling this field to determine an appropriate length. Signed-off-by: Dan McGee --- .../0017_auto__chg_field_mirrorlog_error.py | 66 ++++++++++++++++++++++ mirrors/models.py | 6 +- 2 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py (limited to 'mirrors') diff --git a/mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py b/mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py new file mode 100644 index 00000000..2f76c099 --- /dev/null +++ b/mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.alter_column('mirrors_mirrorlog', 'error', self.gf('django.db.models.fields.TextField')()) + + def backwards(self, orm): + db.alter_column('mirrors_mirrorlog', 'error', self.gf('django.db.models.fields.CharField')(max_length=255)) + + models = { + 'mirrors.mirror': { + 'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + 'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) + }, + 'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) + }, + 'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index 19437610..8c2bd7fc 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -44,6 +44,7 @@ class Mirror(models.Model): def get_absolute_url(self): return '/mirrors/%s/' % self.name + class MirrorProtocol(models.Model): protocol = models.CharField(max_length=10, unique=True) is_download = models.BooleanField(default=True, @@ -57,6 +58,7 @@ class MirrorProtocol(models.Model): class Meta: ordering = ('protocol',) + class MirrorUrl(models.Model): url = models.CharField("URL", max_length=255, unique=True) protocol = models.ForeignKey(MirrorProtocol, related_name="urls", @@ -104,6 +106,7 @@ class MirrorUrl(models.Model): class Meta: verbose_name = 'mirror URL' + class MirrorRsync(models.Model): ip = models.CharField("IP", max_length=24) mirror = models.ForeignKey(Mirror, related_name="rsync_ips") @@ -114,13 +117,14 @@ class MirrorRsync(models.Model): class Meta: verbose_name = 'mirror rsync IP' + class MirrorLog(models.Model): url = models.ForeignKey(MirrorUrl, related_name="logs") check_time = models.DateTimeField(db_index=True) last_sync = models.DateTimeField(null=True) duration = models.FloatField(null=True) is_success = models.BooleanField(default=True) - error = models.CharField(max_length=255, blank=True, default='') + error = models.TextField(blank=True, default='') def __unicode__(self): return "Check of %s at %s" % (self.url.url, self.check_time) -- cgit v1.2.3-2-g168b From 2f7d770b261b3428bcff366ba6ff4fa631dd980a Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 13 May 2012 20:15:00 -0500 Subject: Add rsync support to mirrorcheck and other small improvements The main changes in this patch implement rsync:// protocol checking support by calling the rsync binary, requested in FS#29878. We track and log much of the same things as we already do for FTP and HTTP URLs- check time, last sync, total check duration, etc. Also added in this patch is a configurable timeout value which defaults to the previous hardcoded value of 10 seconds; this can be passed as an option to the mirrorcheck command. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 130 ++++++++++++++++++++--------- 1 file changed, 89 insertions(+), 41 deletions(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index c1269226..ae89d5e0 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -9,22 +9,26 @@ we encounter errors, record those as well. Usage: ./manage.py mirrorcheck """ -from django.core.management.base import NoArgsCommand -from django.db import transaction - from collections import deque from datetime import datetime import logging +import os +from optparse import make_option +from pytz import utc import re import socket +import subprocess import sys import time +import tempfile from threading import Thread import types -from pytz import utc from Queue import Queue, Empty import urllib2 +from django.core.management.base import NoArgsCommand +from django.db import transaction + from main.utils import utc_now from mirrors.models import MirrorUrl, MirrorLog @@ -37,10 +41,15 @@ logger = logging.getLogger() class Command(NoArgsCommand): + option_list = NoArgsCommand.option_list + ( + make_option('-t', '--timeout', dest='timeout', default='10', + help='Timeout value for connecting to URL'), + ) help = "Runs a check on all known mirror URLs to determine their up-to-date status." def handle_noargs(self, **options): v = int(options.get('verbosity', 0)) + timeout = int(options.get('timeout', 10)) if v == 0: logger.level = logging.ERROR elif v == 1: @@ -48,10 +57,29 @@ class Command(NoArgsCommand): elif v == 2: logger.level = logging.DEBUG - return check_current_mirrors() + urls = MirrorUrl.objects.select_related('protocol').filter( + mirror__active=True, mirror__public=True) + + pool = MirrorCheckPool(urls, timeout) + pool.run() + return 0 -def check_mirror_url(mirror_url): +def parse_lastsync(log, data): + '''lastsync file should be an epoch value created by us.''' + try: + parsed_time = datetime.utcfromtimestamp(int(data)) + log.last_sync = parsed_time.replace(tzinfo=utc) + except ValueError: + # it is bad news to try logging the lastsync value; + # sometimes we get a crazy-encoded web page. + # if we couldn't parse a time, this is a failure. + log.last_sync = None + log.error = "Could not parse time from lastsync" + log.is_success = False + + +def check_mirror_url(mirror_url, timeout): url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) log = MirrorLog(url=mirror_url, check_time=utc_now()) @@ -59,28 +87,14 @@ def check_mirror_url(mirror_url): req = urllib2.Request(url, None, headers) try: start = time.time() - result = urllib2.urlopen(req, timeout=10) + result = urllib2.urlopen(req, timeout=timeout) data = result.read() result.close() end = time.time() - # lastsync should be an epoch value created by us - parsed_time = None - try: - parsed_time = datetime.utcfromtimestamp(int(data)) - parsed_time = parsed_time.replace(tzinfo=utc) - except ValueError: - # it is bad news to try logging the lastsync value; - # sometimes we get a crazy-encoded web page. - pass - - log.last_sync = parsed_time - # if we couldn't parse a time, this is a failure - if parsed_time is None: - log.error = "Could not parse time from lastsync" - log.is_success = False + parse_lastsync(log, data) log.duration = end - start logger.debug("success: %s, %.2f", url, log.duration) - except urllib2.HTTPError, e: + except urllib2.HTTPError as e: if e.code == 404: # we have a duration, just not a success end = time.time() @@ -88,7 +102,7 @@ def check_mirror_url(mirror_url): log.is_success = False log.error = str(e) logger.debug("failed: %s, %s", url, log.error) - except urllib2.URLError, e: + except urllib2.URLError as e: log.is_success = False log.error = e.reason if isinstance(e.reason, types.StringTypes) and \ @@ -101,20 +115,64 @@ def check_mirror_url(mirror_url): elif isinstance(e.reason, socket.error): log.error = e.reason.args[1] logger.debug("failed: %s, %s", url, log.error) - except socket.timeout, e: + except socket.timeout as e: log.is_success = False log.error = "Connection timed out." logger.debug("failed: %s, %s", url, log.error) + except socket.error as e: + log.is_success = False + log.error = str(e) + logger.debug("failed: %s, %s", url, log.error) + + return log + + +def check_rsync_url(mirror_url, timeout): + url = mirror_url.url + 'lastsync' + logger.info("checking URL %s", url) + log = MirrorLog(url=mirror_url, check_time=utc_now()) + + tempdir = tempfile.mkdtemp() + lastsync_path = os.path.join(tempdir, 'lastsync') + rsync_cmd = ["rsync", "--quiet", "--contimeout=%d" % timeout, + "--timeout=%d" % timeout, url, lastsync_path] + try: + with open(os.devnull, 'w') as devnull: + proc = subprocess.Popen(rsync_cmd, stdout=devnull, + stderr=subprocess.PIPE) + start = time.time() + _, errdata = proc.communicate() + end = time.time() + log.duration = end - start + if proc.returncode != 0: + logger.debug("error: %s, %s", url, errdata) + log.is_success = False + log.error = errdata.strip() + # look at rsync error code- if we had a command error or timed out, + # don't record a duration as it is misleading + if proc.returncode in (1, 30, 35): + log.duration = None + else: + logger.debug("success: %s, %.2f", url, log.duration) + with open(lastsync_path, 'r') as lastsync: + parse_lastsync(log, lastsync.read()) + finally: + if os.path.exists(lastsync_path): + os.unlink(lastsync_path) + os.rmdir(tempdir) return log -def mirror_url_worker(work, output): +def mirror_url_worker(work, output, timeout): while True: try: - item = work.get(block=False) + url = work.get(block=False) try: - log = check_mirror_url(item) + if url.protocol.protocol == 'rsync': + log = check_rsync_url(url, timeout) + else: + log = check_mirror_url(url, timeout) output.append(log) finally: work.task_done() @@ -123,7 +181,7 @@ def mirror_url_worker(work, output): class MirrorCheckPool(object): - def __init__(self, urls, num_threads=10): + def __init__(self, urls, timeout=10, num_threads=10): self.tasks = Queue() self.logs = deque() for i in list(urls): @@ -131,7 +189,7 @@ class MirrorCheckPool(object): self.threads = [] for i in range(num_threads): thread = Thread(target=mirror_url_worker, - args=(self.tasks, self.logs)) + args=(self.tasks, self.logs, timeout)) thread.daemon = True self.threads.append(thread) @@ -142,18 +200,8 @@ class MirrorCheckPool(object): thread.start() logger.debug("joining on all threads") self.tasks.join() - logger.debug("processing log entries") + logger.debug("processing %d log entries", len(self.logs)) MirrorLog.objects.bulk_create(self.logs) logger.debug("log entries saved") - -def check_current_mirrors(): - urls = MirrorUrl.objects.filter( - protocol__is_download=True, - mirror__active=True, mirror__public=True) - - pool = MirrorCheckPool(urls) - pool.run() - return 0 - # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From ae1c526ffbe908322f0dd8d8805360b81ab22b0f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 13 May 2012 20:35:50 -0500 Subject: Add ability to restrict status report to single tier This should make it easier to catch errors in our Tier 1 mirrors. Signed-off-by: Dan McGee --- mirrors/urls.py | 1 + mirrors/utils.py | 2 +- mirrors/views.py | 19 ++++++++++++++++--- 3 files changed, 18 insertions(+), 4 deletions(-) (limited to 'mirrors') diff --git a/mirrors/urls.py b/mirrors/urls.py index f002e9d6..bb4eb969 100644 --- a/mirrors/urls.py +++ b/mirrors/urls.py @@ -4,6 +4,7 @@ urlpatterns = patterns('mirrors.views', (r'^$', 'mirrors', {}, 'mirror-list'), (r'^status/$', 'status', {}, 'mirror-status'), (r'^status/json/$', 'status_json', {}, 'mirror-status-json'), + (r'^status/tier/(?P\d+)/$', 'status', {}, 'mirror-status-tier'), (r'^(?P[\.\-\w]+)/$', 'mirror_details'), ) diff --git a/mirrors/utils.py b/mirrors/utils.py index 54de567e..2014411d 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -88,7 +88,7 @@ def get_mirror_errors(cutoff=default_cutoff): is_success=False, check_time__gte=cutoff_time, url__mirror__active=True, url__mirror__public=True).values( 'url__url', 'url__country', 'url__protocol__protocol', - 'url__mirror__country', 'error').annotate( + 'url__mirror__country', 'url__mirror__tier', 'error').annotate( error_count=Count('error'), last_occurred=Max('check_time') ).order_by('-last_occurred', '-error_count') errors = list(errors) diff --git a/mirrors/views.py b/mirrors/views.py index b0be6238..8f092be7 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -13,7 +13,7 @@ from django.views.decorators.csrf import csrf_exempt from django.views.generic.simple import direct_to_template from django_countries.countries import COUNTRIES -from .models import Mirror, MirrorUrl, MirrorProtocol +from .models import Mirror, MirrorUrl, MirrorProtocol, TIER_CHOICES from .utils import get_mirror_statuses, get_mirror_errors COUNTRY_LOOKUP = dict(COUNTRIES) @@ -184,7 +184,11 @@ def mirror_details(request, name): {'mirror': mirror, 'urls': all_urls}) -def status(request): +def status(request, tier=None): + if tier is not None: + tier = int(tier) + if tier not in [t[0] for t in TIER_CHOICES]: + raise Http404 bad_timedelta = timedelta(days=3) status_info = get_mirror_statuses() @@ -192,17 +196,26 @@ def status(request): good_urls = [] bad_urls = [] for url in urls: + # screen by tier if we were asked to + if tier is not None and url.mirror.tier != tier: + continue # split them into good and bad lists based on delay if not url.delay or url.delay > bad_timedelta: bad_urls.append(url) else: good_urls.append(url) + error_logs = get_mirror_errors() + if tier is not None: + error_logs = [log for log in error_logs + if log['url__mirror__tier'] == tier] + context = status_info.copy() context.update({ 'good_urls': sorted(good_urls, key=attrgetter('score')), 'bad_urls': sorted(bad_urls, key=lambda u: u.delay or timedelta.max), - 'error_logs': get_mirror_errors(), + 'error_logs': error_logs, + 'tier': tier, }) return direct_to_template(request, 'mirrors/status.html', context) -- cgit v1.2.3-2-g168b From 06f1bb99617e532f6b39c135370de79be7c270fa Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 18 May 2012 20:42:05 -0500 Subject: mirrors: add an alternate_email column We have a lot of these in the freeform text area in the mirror notes; attempt to make this data usable as necessary if we want to do some sort of mirror notification automation in the future. Signed-off-by: Dan McGee --- mirrors/admin.py | 4 +- .../0018_auto__add_field_mirror_alternate_email.py | 68 ++++++++++++++++++++++ mirrors/models.py | 1 + 3 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 mirrors/migrations/0018_auto__add_field_mirror_alternate_email.py (limited to 'mirrors') diff --git a/mirrors/admin.py b/mirrors/admin.py index b7b9894c..65fff368 100644 --- a/mirrors/admin.py +++ b/mirrors/admin.py @@ -63,9 +63,9 @@ class MirrorAdminForm(forms.ModelForm): class MirrorAdmin(admin.ModelAdmin): form = MirrorAdminForm list_display = ('name', 'tier', 'country', 'active', 'public', - 'isos', 'admin_email') + 'isos', 'admin_email', 'alternate_email') list_filter = ('tier', 'active', 'public', 'country') - search_fields = ('name',) + search_fields = ('name', 'admin_email', 'alternate_email') inlines = [ MirrorUrlInlineAdmin, MirrorRsyncInlineAdmin, diff --git a/mirrors/migrations/0018_auto__add_field_mirror_alternate_email.py b/mirrors/migrations/0018_auto__add_field_mirror_alternate_email.py new file mode 100644 index 00000000..a08699e8 --- /dev/null +++ b/mirrors/migrations/0018_auto__add_field_mirror_alternate_email.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + def forwards(self, orm): + db.add_column('mirrors_mirror', 'alternate_email', + self.gf('django.db.models.fields.EmailField')(default='', max_length=255, blank=True), + keep_default=False) + + def backwards(self, orm): + db.delete_column('mirrors_mirror', 'alternate_email') + + models = { + 'mirrors.mirror': { + 'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + 'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) + }, + 'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) + }, + 'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index 8c2bd7fc..9a545b51 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -20,6 +20,7 @@ class Mirror(models.Model): upstream = models.ForeignKey('self', null=True, on_delete=models.SET_NULL) country = CountryField(blank=True, db_index=True) admin_email = models.EmailField(max_length=255, blank=True) + alternate_email = models.EmailField(max_length=255, blank=True) public = models.BooleanField(default=True) active = models.BooleanField(default=True) isos = models.BooleanField("ISOs", default=True) -- cgit v1.2.3-2-g168b From a87da032cb6b5b84624e4205b5f8b7cab37249cd Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 8 Jul 2012 20:36:51 -0500 Subject: Handle HTTPException being thrown in mirrorcheck Managed to see this bubble up today when running the mirrorcheck command on a less than ideal connection that was experiencing timeouts at the wrong time. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index ae89d5e0..3d431796 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -11,6 +11,7 @@ Usage: ./manage.py mirrorcheck from collections import deque from datetime import datetime +from httplib import HTTPException import logging import os from optparse import make_option @@ -115,6 +116,11 @@ def check_mirror_url(mirror_url, timeout): elif isinstance(e.reason, socket.error): log.error = e.reason.args[1] logger.debug("failed: %s, %s", url, log.error) + except HTTPException as e: + # e.g., BadStatusLine + log.is_success = False + log.error = "Exception in processing HTTP request." + logger.debug("failed: %s, %s", url, log.error) except socket.timeout as e: log.is_success = False log.error = "Connection timed out." -- cgit v1.2.3-2-g168b From 0f3c894e7a0f573fa0198459150f387c3a7f23ae Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 8 Jul 2012 20:38:01 -0500 Subject: Don't include StdDev on sqlite3 mirror status query Because this function isn't shipped by default, it makes more sense to just omit it completely from the query we do to build the tables on this page when in development. Substitute 0.0 for the value so the rest of the calculations and display work as expected. Signed-off-by: Dan McGee --- mirrors/utils.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index 2014411d..9aa8e0f5 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -3,7 +3,7 @@ from datetime import timedelta from django.db.models import Avg, Count, Max, Min, StdDev from django_countries.fields import Country -from main.utils import cache_function, utc_now +from main.utils import cache_function, utc_now, database_vendor from .models import MirrorLog, MirrorProtocol, MirrorUrl @@ -40,8 +40,11 @@ def get_mirror_statuses(cutoff=default_cutoff): success_count=Count('logs__duration'), last_sync=Max('logs__last_sync'), last_check=Max('logs__check_time'), - duration_avg=Avg('logs__duration'), - duration_stddev=StdDev('logs__duration')) + duration_avg=Avg('logs__duration')) + + vendor = database_vendor(MirrorUrl) + if vendor != 'sqlite': + urls.annotate(duration_stddev=StdDev('logs__duration')) # The Django ORM makes it really hard to get actual average delay in the # above query, so run a seperate query for it and we will process the @@ -70,6 +73,9 @@ def get_mirror_statuses(cutoff=default_cutoff): check_frequency = None for url in urls: + # fake the standard deviation for local testing setups + if vendor == 'sqlite': + setattr(url, 'duration_stddev', 0.0) annotate_url(url, delays) return { -- cgit v1.2.3-2-g168b From 3c4ceb16331b37fd334dc9682d4cde6430838942 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 8 Jul 2012 20:56:28 -0500 Subject: mirrorcheck: Don't use bulk_create on sqlite3 It isn't worth it, as we run into the 999 max SQL statement variables issue when using it on any significant amount of mirrors. Since this is just a development database setup, and it isn't a command we need to run especially fast, we can ditch it. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 3d431796..7a133cbf 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -30,7 +30,7 @@ import urllib2 from django.core.management.base import NoArgsCommand from django.db import transaction -from main.utils import utc_now +from main.utils import utc_now, database_vendor from mirrors.models import MirrorUrl, MirrorLog logging.basicConfig( @@ -207,7 +207,11 @@ class MirrorCheckPool(object): logger.debug("joining on all threads") self.tasks.join() logger.debug("processing %d log entries", len(self.logs)) - MirrorLog.objects.bulk_create(self.logs) + if database_vendor(MirrorLog, mode='write') == 'sqlite': + for log in self.logs: + log.save(force_insert=True) + else: + MirrorLog.objects.bulk_create(self.logs) logger.debug("log entries saved") # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From 9c7350650e66b5eb6228778e335a160be5ea7f98 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 8 Jul 2012 21:53:35 -0500 Subject: Correctly reassign queryset with added annotation in mirror status This was a dumb oversight on my part in commit 0f3c894e7a0. Signed-off-by: Dan McGee --- mirrors/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index 9aa8e0f5..f2c98ee0 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -44,7 +44,7 @@ def get_mirror_statuses(cutoff=default_cutoff): vendor = database_vendor(MirrorUrl) if vendor != 'sqlite': - urls.annotate(duration_stddev=StdDev('logs__duration')) + urls = urls.annotate(duration_stddev=StdDev('logs__duration')) # The Django ORM makes it really hard to get actual average delay in the # above query, so run a seperate query for it and we will process the -- cgit v1.2.3-2-g168b From c0bf9e20660cfae7ea8994472555bba23398b598 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 24 Jul 2012 09:19:48 -0500 Subject: Remove custom utc_now() function, use django.utils.timezone.now() This was around from the time when we handled timezones sanely and Django did not; now that we are on 1.4 we no longer need our own code to handle this. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 7 ++++--- mirrors/utils.py | 17 +++++++++-------- 2 files changed, 13 insertions(+), 11 deletions(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 7a133cbf..e09ea680 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -29,8 +29,9 @@ import urllib2 from django.core.management.base import NoArgsCommand from django.db import transaction +from django.utils.timezone import now -from main.utils import utc_now, database_vendor +from main.utils import database_vendor from mirrors.models import MirrorUrl, MirrorLog logging.basicConfig( @@ -83,7 +84,7 @@ def parse_lastsync(log, data): def check_mirror_url(mirror_url, timeout): url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) - log = MirrorLog(url=mirror_url, check_time=utc_now()) + log = MirrorLog(url=mirror_url, check_time=now()) headers = {'User-Agent': 'archweb/1.0'} req = urllib2.Request(url, None, headers) try: @@ -136,7 +137,7 @@ def check_mirror_url(mirror_url, timeout): def check_rsync_url(mirror_url, timeout): url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) - log = MirrorLog(url=mirror_url, check_time=utc_now()) + log = MirrorLog(url=mirror_url, check_time=now()) tempdir = tempfile.mkdtemp() lastsync_path = os.path.join(tempdir, 'lastsync') diff --git a/mirrors/utils.py b/mirrors/utils.py index f2c98ee0..bf030d39 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -1,13 +1,14 @@ from datetime import timedelta from django.db.models import Avg, Count, Max, Min, StdDev +from django.utils.timezone import now from django_countries.fields import Country -from main.utils import cache_function, utc_now, database_vendor +from main.utils import cache_function, database_vendor from .models import MirrorLog, MirrorProtocol, MirrorUrl -default_cutoff = timedelta(hours=24) +DEFAULT_CUTOFF = timedelta(hours=24) def annotate_url(url, delays): '''Given a MirrorURL object, add a few more attributes to it regarding @@ -30,8 +31,8 @@ def annotate_url(url, delays): @cache_function(123) -def get_mirror_statuses(cutoff=default_cutoff): - cutoff_time = utc_now() - cutoff +def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): + cutoff_time = now() - cutoff # I swear, this actually has decent performance... urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( mirror__active=True, mirror__public=True, @@ -88,8 +89,8 @@ def get_mirror_statuses(cutoff=default_cutoff): @cache_function(117) -def get_mirror_errors(cutoff=default_cutoff): - cutoff_time = utc_now() - cutoff +def get_mirror_errors(cutoff=DEFAULT_CUTOFF): + cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( is_success=False, check_time__gte=cutoff_time, url__mirror__active=True, url__mirror__public=True).values( @@ -105,11 +106,11 @@ def get_mirror_errors(cutoff=default_cutoff): @cache_function(295) -def get_mirror_url_for_download(cutoff=default_cutoff): +def get_mirror_url_for_download(cutoff=DEFAULT_CUTOFF): '''Find a good mirror URL to use for package downloads. If we have mirror status data available, it is used to determine a good choice by looking at the last batch of status rows.''' - cutoff_time = utc_now() - cutoff + cutoff_time = now() - cutoff status_data = MirrorLog.objects.filter( check_time__gte=cutoff_time).aggregate( Max('check_time'), Max('last_sync')) -- cgit v1.2.3-2-g168b From 76c37ce3acc7a4af0271c7535d4a33042f7749b5 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 24 Jul 2012 09:35:55 -0500 Subject: Replace deprecated direct_to_template() with render() shortcut Now that Django actually provides a concise way to use a RequestContext object without instantiating it, we can use that rather than the old function-based generic view that worked well to do the same. Additionally, these function-based generic views will be gone in Django 1.5, so might as well make the move now. Signed-off-by: Dan McGee --- mirrors/views.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) (limited to 'mirrors') diff --git a/mirrors/views.py b/mirrors/views.py index 8f092be7..400c084d 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -8,9 +8,8 @@ from django.forms.widgets import CheckboxSelectMultiple from django.core.serializers.json import DjangoJSONEncoder from django.db.models import Q from django.http import Http404, HttpResponse -from django.shortcuts import get_object_or_404 +from django.shortcuts import get_object_or_404, render from django.views.decorators.csrf import csrf_exempt -from django.views.generic.simple import direct_to_template from django_countries.countries import COUNTRIES from .models import Mirror, MirrorUrl, MirrorProtocol, TIER_CHOICES @@ -76,7 +75,7 @@ def generate_mirrorlist(request): else: form = MirrorlistForm() - return direct_to_template(request, 'mirrors/mirrorlist_generate.html', + return render(request, 'mirrors/mirrorlist_generate.html', {'mirrorlist_form': form}) @@ -142,10 +141,10 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, urls = status_filter(urls) template = 'mirrors/mirrorlist_status.txt' - return direct_to_template(request, template, { - 'mirror_urls': urls, - }, - mimetype='text/plain') + context = { + 'mirror_urls': urls, + } + return render(request, template, context, content_type='text/plain') def find_mirrors_simple(request, protocol): @@ -161,7 +160,7 @@ def mirrors(request): mirror_list = Mirror.objects.select_related().order_by('tier', 'country') if not request.user.is_authenticated(): mirror_list = mirror_list.filter(public=True, active=True) - return direct_to_template(request, 'mirrors/mirrors.html', + return render(request, 'mirrors/mirrors.html', {'mirror_list': mirror_list}) @@ -180,7 +179,7 @@ def mirror_details(request, name): all_urls = set(checked_urls).union(all_urls) all_urls = sorted(all_urls, key=attrgetter('url')) - return direct_to_template(request, 'mirrors/mirror_details.html', + return render(request, 'mirrors/mirror_details.html', {'mirror': mirror, 'urls': all_urls}) @@ -217,7 +216,7 @@ def status(request, tier=None): 'error_logs': error_logs, 'tier': tier, }) - return direct_to_template(request, 'mirrors/status.html', context) + return render(request, 'mirrors/status.html', context) class MirrorStatusJSONEncoder(DjangoJSONEncoder): -- cgit v1.2.3-2-g168b From 686942b8788fa43031b3999ac00d60baadc82f53 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 31 Jul 2012 19:56:49 -0500 Subject: Declare 'enums' at class scope Signed-off-by: Dan McGee --- mirrors/models.py | 15 +++++++-------- mirrors/views.py | 4 ++-- 2 files changed, 9 insertions(+), 10 deletions(-) (limited to 'mirrors') diff --git a/mirrors/models.py b/mirrors/models.py index 9a545b51..06b483d5 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -6,15 +6,14 @@ from django.core.exceptions import ValidationError from django_countries import CountryField -TIER_CHOICES = ( - (0, 'Tier 0'), - (1, 'Tier 1'), - (2, 'Tier 2'), - (-1, 'Untiered'), -) - - class Mirror(models.Model): + TIER_CHOICES = ( + (0, 'Tier 0'), + (1, 'Tier 1'), + (2, 'Tier 2'), + (-1, 'Untiered'), + ) + name = models.CharField(max_length=255, unique=True) tier = models.SmallIntegerField(default=2, choices=TIER_CHOICES) upstream = models.ForeignKey('self', null=True, on_delete=models.SET_NULL) diff --git a/mirrors/views.py b/mirrors/views.py index 400c084d..2c2577f4 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -12,7 +12,7 @@ from django.shortcuts import get_object_or_404, render from django.views.decorators.csrf import csrf_exempt from django_countries.countries import COUNTRIES -from .models import Mirror, MirrorUrl, MirrorProtocol, TIER_CHOICES +from .models import Mirror, MirrorUrl, MirrorProtocol from .utils import get_mirror_statuses, get_mirror_errors COUNTRY_LOOKUP = dict(COUNTRIES) @@ -186,7 +186,7 @@ def mirror_details(request, name): def status(request, tier=None): if tier is not None: tier = int(tier) - if tier not in [t[0] for t in TIER_CHOICES]: + if tier not in [t[0] for t in Mirror.TIER_CHOICES]: raise Http404 bad_timedelta = timedelta(days=3) status_info = get_mirror_statuses() -- cgit v1.2.3-2-g168b From b7a03d89d126989bf53005404759482e17163991 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 7 Aug 2012 23:22:18 -0500 Subject: Push more default values down into the database This makes it easier to do manual manipulation/insertion/etc. at the database level, as well as just making things act more sane from an overall software stack perspective. Signed-off-by: Dan McGee --- mirrors/migrations/0004_auto__add_field_mirrorprotocol_is_download.py | 2 +- ...uto__add_field_mirrorurl_has_ipv4__add_field_mirrorurl_has_ipv6.py | 4 ++-- mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py | 2 +- mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) (limited to 'mirrors') diff --git a/mirrors/migrations/0004_auto__add_field_mirrorprotocol_is_download.py b/mirrors/migrations/0004_auto__add_field_mirrorprotocol_is_download.py index 5e44d211..0506e2cd 100644 --- a/mirrors/migrations/0004_auto__add_field_mirrorprotocol_is_download.py +++ b/mirrors/migrations/0004_auto__add_field_mirrorprotocol_is_download.py @@ -7,7 +7,7 @@ from django.db import models class Migration(SchemaMigration): def forwards(self, orm): - db.add_column('mirrors_mirrorprotocol', 'is_download', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=False) + db.add_column('mirrors_mirrorprotocol', 'is_download', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=True) def backwards(self, orm): db.delete_column('mirrors_mirrorprotocol', 'is_download') diff --git a/mirrors/migrations/0006_auto__add_field_mirrorurl_has_ipv4__add_field_mirrorurl_has_ipv6.py b/mirrors/migrations/0006_auto__add_field_mirrorurl_has_ipv4__add_field_mirrorurl_has_ipv6.py index a5e34589..5a40207d 100644 --- a/mirrors/migrations/0006_auto__add_field_mirrorurl_has_ipv4__add_field_mirrorurl_has_ipv6.py +++ b/mirrors/migrations/0006_auto__add_field_mirrorurl_has_ipv4__add_field_mirrorurl_has_ipv6.py @@ -7,8 +7,8 @@ from django.db import models class Migration(SchemaMigration): def forwards(self, orm): - db.add_column('mirrors_mirrorurl', 'has_ipv4', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=False) - db.add_column('mirrors_mirrorurl', 'has_ipv6', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=False) + db.add_column('mirrors_mirrorurl', 'has_ipv4', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=True) + db.add_column('mirrors_mirrorurl', 'has_ipv6', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=True) def backwards(self, orm): db.delete_column('mirrors_mirrorurl', 'has_ipv4') diff --git a/mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py b/mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py index d30c78c7..66e60090 100644 --- a/mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py +++ b/mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py @@ -6,7 +6,7 @@ from django.db import models class Migration(SchemaMigration): def forwards(self, orm): - db.add_column('mirrors_mirrorprotocol', 'default', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=False) + db.add_column('mirrors_mirrorprotocol', 'default', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=True) def backwards(self, orm): db.delete_column('mirrors_mirrorprotocol', 'default') diff --git a/mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py b/mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py index 2f76c099..60c4ec26 100644 --- a/mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py +++ b/mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py @@ -7,7 +7,7 @@ from django.db import models class Migration(SchemaMigration): def forwards(self, orm): - db.alter_column('mirrors_mirrorlog', 'error', self.gf('django.db.models.fields.TextField')()) + db.alter_column('mirrors_mirrorlog', 'error', self.gf('django.db.models.fields.TextField')(default='')) def backwards(self, orm): db.alter_column('mirrors_mirrorlog', 'error', self.gf('django.db.models.fields.CharField')(max_length=255)) -- cgit v1.2.3-2-g168b From e9c4985538c067a09a186967f77c5395fb60675b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 18 Sep 2012 22:10:54 -0500 Subject: Sort mirrorlist by English country name, not ISO code Fixes FS#31503. Signed-off-by: Dan McGee --- mirrors/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/views.py b/mirrors/views.py index 2c2577f4..11719223 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -134,7 +134,7 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, urls = qset if not use_status: - sort_key = attrgetter('real_country', 'mirror.name', 'url') + sort_key = attrgetter('real_country.name', 'mirror.name', 'url') urls = sorted(urls, key=sort_key) template = 'mirrors/mirrorlist.txt' else: -- cgit v1.2.3-2-g168b From f0b7e73de61c03a5018ed352605e6329611448d2 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 10 Oct 2012 20:17:55 -0500 Subject: Make mirror log time query a bit more efficient We don't need the full mirror log objects; we just need a very small subset of values from them here to do the required math and object building. Signed-off-by: Dan McGee --- mirrors/utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index bf030d39..0a32b766 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -50,12 +50,14 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): # The Django ORM makes it really hard to get actual average delay in the # above query, so run a seperate query for it and we will process the # results here. - times = MirrorLog.objects.filter(is_success=True, last_sync__isnull=False, + times = MirrorLog.objects.values_list( + 'url_id', 'check_time', 'last_sync').filter( + is_success=True, last_sync__isnull=False, check_time__gte=cutoff_time) delays = {} - for log in times: - delay = log.check_time - log.last_sync - delays.setdefault(log.url_id, []).append(delay) + for url_id, check_time, last_sync in times: + delay = check_time - last_sync + delays.setdefault(url_id, []).append(delay) if urls: last_check = max([u.last_check for u in urls]) -- cgit v1.2.3-2-g168b From 4ab5d6947795f1fef0d38601ec7ad3ca5f62173e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 10 Nov 2012 14:13:34 -0600 Subject: Add mirror extended status JSON view When asking for status for a single mirror, we can include logs from the past 24 hours in addition to the normal information we provide. This is slated for usage by a frontend graph still to come, similar to those on the NTP pool website. Signed-off-by: Dan McGee --- mirrors/urls.py | 1 + mirrors/views.py | 37 ++++++++++++++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 3 deletions(-) (limited to 'mirrors') diff --git a/mirrors/urls.py b/mirrors/urls.py index bb4eb969..857e99e2 100644 --- a/mirrors/urls.py +++ b/mirrors/urls.py @@ -6,6 +6,7 @@ urlpatterns = patterns('mirrors.views', (r'^status/json/$', 'status_json', {}, 'mirror-status-json'), (r'^status/tier/(?P\d+)/$', 'status', {}, 'mirror-status-tier'), (r'^(?P[\.\-\w]+)/$', 'mirror_details'), + (r'^(?P[\.\-\w]+)/json/$', 'mirror_details_json'), ) # vim: set ts=4 sw=4 et: diff --git a/mirrors/views.py b/mirrors/views.py index 11719223..cbd86611 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -9,10 +9,11 @@ from django.core.serializers.json import DjangoJSONEncoder from django.db.models import Q from django.http import Http404, HttpResponse from django.shortcuts import get_object_or_404, render +from django.utils.timezone import now from django.views.decorators.csrf import csrf_exempt from django_countries.countries import COUNTRIES -from .models import Mirror, MirrorUrl, MirrorProtocol +from .models import Mirror, MirrorUrl, MirrorProtocol, MirrorLog from .utils import get_mirror_statuses, get_mirror_errors COUNTRY_LOOKUP = dict(COUNTRIES) @@ -183,6 +184,19 @@ def mirror_details(request, name): {'mirror': mirror, 'urls': all_urls}) +def mirror_details_json(request, name): + mirror = get_object_or_404(Mirror, name=name) + status_info = get_mirror_statuses() + data = status_info.copy() + data['version'] = 3 + # include only URLs for this particular mirror + data['urls'] = [url for url in data['urls'] if url.mirror_id == mirror.id] + to_json = json.dumps(data, ensure_ascii=False, + cls=ExtendedMirrorStatusJSONEncoder) + response = HttpResponse(to_json, mimetype='application/json') + return response + + def status(request, tier=None): if tier is not None: tier = int(tier) @@ -222,8 +236,8 @@ def status(request, tier=None): class MirrorStatusJSONEncoder(DjangoJSONEncoder): '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl serialization. The base class takes care of datetime.datetime types.''' - url_attributes = ['url', 'protocol', 'last_sync', 'completion_pct', - 'delay', 'duration_avg', 'duration_stddev', 'score'] + url_attributes = ('url', 'protocol', 'last_sync', 'completion_pct', + 'delay', 'duration_avg', 'duration_stddev', 'score') def default(self, obj): if isinstance(obj, timedelta): @@ -245,6 +259,23 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder): return super(MirrorStatusJSONEncoder, self).default(obj) +class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): + '''Adds URL check history information.''' + log_attributes = ('check_time', 'last_sync', 'duration', 'is_success') + + def default(self, obj): + if isinstance(obj, MirrorUrl): + data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj) + cutoff = now() - timedelta(hours=24) + data['logs'] = obj.logs.filter(check_time__gte=cutoff) + return data + if isinstance(obj, MirrorLog): + data = dict((attr, getattr(obj, attr)) + for attr in self.log_attributes) + return data + return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) + + def status_json(request): status_info = get_mirror_statuses() data = status_info.copy() -- cgit v1.2.3-2-g168b From 86102c6e645451c03e3e576060eba7f93350bf6b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 10 Nov 2012 14:19:23 -0600 Subject: Allow filtering retrieved mirror statuses by mirror_id When we don't need them all, no need to fetch them all. Let the database do the work for us, hopefully. Signed-off-by: Dan McGee --- mirrors/utils.py | 19 +++++++++++++++---- mirrors/views.py | 8 ++++---- 2 files changed, 19 insertions(+), 8 deletions(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index 0a32b766..ba027c99 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -31,7 +31,7 @@ def annotate_url(url, delays): @cache_function(123) -def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): +def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): cutoff_time = now() - cutoff # I swear, this actually has decent performance... urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( @@ -43,6 +43,9 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): last_check=Max('logs__check_time'), duration_avg=Avg('logs__duration')) + if mirror_ids: + urls = urls.filter(mirror_id__in=mirror_ids) + vendor = database_vendor(MirrorUrl) if vendor != 'sqlite': urls = urls.annotate(duration_stddev=StdDev('logs__duration')) @@ -54,6 +57,8 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): 'url_id', 'check_time', 'last_sync').filter( is_success=True, last_sync__isnull=False, check_time__gte=cutoff_time) + if mirror_ids: + times = times.filter(url__mirror_id__in=mirror_ids) delays = {} for url_id, check_time, last_sync in times: delay = check_time - last_sync @@ -62,8 +67,10 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): if urls: last_check = max([u.last_check for u in urls]) num_checks = max([u.check_count for u in urls]) - check_info = MirrorLog.objects.filter( - check_time__gte=cutoff_time).aggregate( + check_info = MirrorLog.objects.filter(check_time__gte=cutoff_time) + if mirror_ids: + check_info = check_info.filter(url__mirror_id__in=mirror_ids) + check_info = check_info.aggregate( mn=Min('check_time'), mx=Max('check_time')) if num_checks > 1: check_frequency = (check_info['mx'] - check_info['mn']) \ @@ -91,7 +98,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): @cache_function(117) -def get_mirror_errors(cutoff=DEFAULT_CUTOFF): +def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None): cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( is_success=False, check_time__gte=cutoff_time, @@ -100,6 +107,10 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF): 'url__mirror__country', 'url__mirror__tier', 'error').annotate( error_count=Count('error'), last_occurred=Max('check_time') ).order_by('-last_occurred', '-error_count') + + if mirror_ids: + urls = urls.filter(mirror_id__in=mirror_ids) + errors = list(errors) for err in errors: ctry_code = err['url__country'] or err['url__mirror__country'] diff --git a/mirrors/views.py b/mirrors/views.py index cbd86611..4b9721dc 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -14,7 +14,7 @@ from django.views.decorators.csrf import csrf_exempt from django_countries.countries import COUNTRIES from .models import Mirror, MirrorUrl, MirrorProtocol, MirrorLog -from .utils import get_mirror_statuses, get_mirror_errors +from .utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF COUNTRY_LOOKUP = dict(COUNTRIES) @@ -171,7 +171,7 @@ def mirror_details(request, name): (not mirror.public or not mirror.active): raise Http404 - status_info = get_mirror_statuses() + status_info = get_mirror_statuses(mirror_ids=[mirror.id]) checked_urls = [url for url in status_info['urls'] \ if url.mirror_id == mirror.id] all_urls = mirror.urls.select_related('protocol') @@ -186,7 +186,7 @@ def mirror_details(request, name): def mirror_details_json(request, name): mirror = get_object_or_404(Mirror, name=name) - status_info = get_mirror_statuses() + status_info = get_mirror_statuses(mirror_ids=[mirror.id]) data = status_info.copy() data['version'] = 3 # include only URLs for this particular mirror @@ -266,7 +266,7 @@ class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): def default(self, obj): if isinstance(obj, MirrorUrl): data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj) - cutoff = now() - timedelta(hours=24) + cutoff = now() - DEFAULT_CUTOFF data['logs'] = obj.logs.filter(check_time__gte=cutoff) return data if isinstance(obj, MirrorLog): -- cgit v1.2.3-2-g168b From 07d2fc5d358992a52908cccbbca4a11d01e98da3 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 10 Nov 2012 15:41:09 -0600 Subject: Add initial version of mirror status chart Still have some hardcoded stuff to rip out and replace to make this a bit more dynamic on things like sizing, but for now, this is a great start. Signed-off-by: Dan McGee --- mirrors/static/mirror_status.js | 97 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 mirrors/static/mirror_status.js (limited to 'mirrors') diff --git a/mirrors/static/mirror_status.js b/mirrors/static/mirror_status.js new file mode 100644 index 00000000..1c510cee --- /dev/null +++ b/mirrors/static/mirror_status.js @@ -0,0 +1,97 @@ +function mirror_status(chart_id, data_url) { + d3.json(data_url, function(json) { + data = jQuery.map(json['urls'], + function(url, i) { + return jQuery.map(url['logs'], + function(log, j) { + return { + url: url['url'], + duration: log['duration'], + check_time: new Date(log['check_time']) + }; + }); + }); + + var margin = {top: 20, right: 20, bottom: 30, left: 40}, + width = 1200 - margin.left - margin.right, + height = 450 - margin.top - margin.bottom; + + var color = d3.scale.category20(), + x = d3.time.scale.utc().range([0, width]), + y = d3.scale.linear().range([height, 0]), + x_axis = d3.svg.axis().scale(x).orient("bottom"), + y_axis = d3.svg.axis().scale(y).orient("left"); + + var svg = d3.select(chart_id).append("svg") + .attr("width", width + margin.left + margin.right) + .attr("height", height + margin.top + margin.bottom) + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + + x.domain(d3.extent(data, function(d) { return d.check_time; })).nice(d3.time.hour); + y.domain(d3.extent(data, function(d) { return d.duration; })).nice(); + + svg.append("g") + .attr("class", "x axis") + .attr("transform", "translate(0," + height + ")") + .call(x_axis) + .append("text") + .attr("class", "label") + .attr("x", width) + .attr("y", -6) + .style("text-anchor", "end") + .text("Check Time (UTC)"); + + svg.append("g") + .attr("class", "y axis") + .call(y_axis) + .append("text") + .attr("class", "label") + .attr("transform", "rotate(-90)") + .attr("y", 6) + .attr("dy", ".71em") + .style("text-anchor", "end") + .text("Duration (seconds)"); + + svg.selectAll(".dot") + .data(data) + .enter() + .append("circle") + .attr("class", "dot") + .attr("r", 3.5) + .attr("cx", function(d) { return x(d.check_time); }) + .attr("cy", function(d) { return y(d.duration); }) + .style("fill", function(d) { return color(d.url); }); + + var legend = svg.selectAll(".legend") + .data(color.domain()) + .enter().append("g") + .attr("class", "legend") + .attr("transform", function(d, i) { return "translate(0," + i * 20 + ")"; }); + + legend.append("rect") + .attr("x", width - 18) + .attr("width", 18) + .attr("height", 18) + .style("fill", color); + + legend.append("text") + .attr("x", width - 24) + .attr("y", 9) + .attr("dy", ".35em") + .style("text-anchor", "end") + .text(function(d) { return d; }); + }); + + var resize_timeout = null; + var real_resize = function() { + resize_timeout = null; + /* TODO: implement resize */ + }; + jQuery(window).resize(function() { + if (resize_timeout) { + clearTimeout(resize_timeout); + } + resize_timeout = setTimeout(real_resize, 200); + }); +} -- cgit v1.2.3-2-g168b From aeeb4718e83cd2f82d94b1aa0c0ba36ba21a2b37 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 10 Nov 2012 16:12:15 -0600 Subject: Enable mirror status graph resizing Signed-off-by: Dan McGee --- mirrors/static/mirror_status.js | 43 +++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 15 deletions(-) (limited to 'mirrors') diff --git a/mirrors/static/mirror_status.js b/mirrors/static/mirror_status.js index 1c510cee..277d3c88 100644 --- a/mirrors/static/mirror_status.js +++ b/mirrors/static/mirror_status.js @@ -1,20 +1,10 @@ function mirror_status(chart_id, data_url) { - d3.json(data_url, function(json) { - data = jQuery.map(json['urls'], - function(url, i) { - return jQuery.map(url['logs'], - function(log, j) { - return { - url: url['url'], - duration: log['duration'], - check_time: new Date(log['check_time']) - }; - }); - }); + var jq_div = jQuery(chart_id); + var draw_graph = function(data) { var margin = {top: 20, right: 20, bottom: 30, left: 40}, - width = 1200 - margin.left - margin.right, - height = 450 - margin.top - margin.bottom; + width = jq_div.width() - margin.left - margin.right, + height = jq_div.height() - margin.top - margin.bottom; var color = d3.scale.category20(), x = d3.time.scale.utc().range([0, width]), @@ -22,6 +12,8 @@ function mirror_status(chart_id, data_url) { x_axis = d3.svg.axis().scale(x).orient("bottom"), y_axis = d3.svg.axis().scale(y).orient("left"); + /* remove any existing graph first if we are redrawing after resize */ + d3.select(chart_id).select("svg").remove(); var svg = d3.select(chart_id).append("svg") .attr("width", width + margin.left + margin.right) .attr("height", height + margin.top + margin.bottom) @@ -31,6 +23,7 @@ function mirror_status(chart_id, data_url) { x.domain(d3.extent(data, function(d) { return d.check_time; })).nice(d3.time.hour); y.domain(d3.extent(data, function(d) { return d.duration; })).nice(); + /* build the axis lines... */ svg.append("g") .attr("class", "x axis") .attr("transform", "translate(0," + height + ")") @@ -53,6 +46,7 @@ function mirror_status(chart_id, data_url) { .style("text-anchor", "end") .text("Duration (seconds)"); + /* ...then the points themselves. */ svg.selectAll(".dot") .data(data) .enter() @@ -63,6 +57,7 @@ function mirror_status(chart_id, data_url) { .attr("cy", function(d) { return y(d.duration); }) .style("fill", function(d) { return color(d.url); }); + /* add a legend for good measure */ var legend = svg.selectAll(".legend") .data(color.domain()) .enter().append("g") @@ -81,12 +76,30 @@ function mirror_status(chart_id, data_url) { .attr("dy", ".35em") .style("text-anchor", "end") .text(function(d) { return d; }); + }; + + /* invoke the data-fetch + first draw */ + var cached_data = null; + d3.json(data_url, function(json) { + cached_data = jQuery.map(json['urls'], + function(url, i) { + return jQuery.map(url['logs'], + function(log, j) { + return { + url: url['url'], + duration: log['duration'], + check_time: new Date(log['check_time']) + }; + }); + }); + draw_graph(cached_data); }); + /* then hook up a resize handler to redraw if necessary */ var resize_timeout = null; var real_resize = function() { resize_timeout = null; - /* TODO: implement resize */ + draw_graph(cached_data); }; jQuery(window).resize(function() { if (resize_timeout) { -- cgit v1.2.3-2-g168b From a358e132886972dc4e9f1f546e36a5f3a2218a39 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 10 Nov 2012 17:09:51 -0600 Subject: Mirror status graph, now with points AND lines We might have to tweak the interpolation method once we see this with real data, but for now it looks really pretty locally. Signed-off-by: Dan McGee --- mirrors/static/mirror_status.js | 58 +++++++++++++++++++++++++++++------------ mirrors/views.py | 3 ++- 2 files changed, 44 insertions(+), 17 deletions(-) (limited to 'mirrors') diff --git a/mirrors/static/mirror_status.js b/mirrors/static/mirror_status.js index 277d3c88..d107d7d1 100644 --- a/mirrors/static/mirror_status.js +++ b/mirrors/static/mirror_status.js @@ -6,7 +6,7 @@ function mirror_status(chart_id, data_url) { width = jq_div.width() - margin.left - margin.right, height = jq_div.height() - margin.top - margin.bottom; - var color = d3.scale.category20(), + var color = d3.scale.category10(), x = d3.time.scale.utc().range([0, width]), y = d3.scale.linear().range([height, 0]), x_axis = d3.svg.axis().scale(x).orient("bottom"), @@ -20,8 +20,14 @@ function mirror_status(chart_id, data_url) { .append("g") .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); - x.domain(d3.extent(data, function(d) { return d.check_time; })).nice(d3.time.hour); - y.domain(d3.extent(data, function(d) { return d.duration; })).nice(); + x.domain([ + d3.min(data, function(c) { return d3.min(c.logs, function(v) { return v.check_time; }); }), + d3.max(data, function(c) { return d3.max(c.logs, function(v) { return v.check_time; }); }) + ]); + y.domain([ + d3.min(data, function(c) { return d3.min(c.logs, function(v) { return v.duration; }); }), + d3.max(data, function(c) { return d3.max(c.logs, function(v) { return v.duration; }); }) + ]); /* build the axis lines... */ svg.append("g") @@ -46,12 +52,32 @@ function mirror_status(chart_id, data_url) { .style("text-anchor", "end") .text("Duration (seconds)"); - /* ...then the points themselves. */ - svg.selectAll(".dot") + var line = d3.svg.line() + .interpolate("basis") + .x(function(d) { return x(d.check_time); }) + .y(function(d) { return y(d.duration); }); + + /* ...then the points and lines between them. */ + var urls = svg.selectAll(".url") .data(data) .enter() + .append("g") + .attr("class", "url"); + + urls.append("path") + .attr("class", "url-line") + .attr("d", function(d) { return line(d.logs); }) + .style("stroke", function(d) { return color(d.url); }); + + urls.selectAll("circle") + .data(function(u) { + return jQuery.map(u.logs, function(l, i) { + return {url: u.url, check_time: l.check_time, duration: l.duration}; + }); + }) + .enter() .append("circle") - .attr("class", "dot") + .attr("class", "url-dot") .attr("r", 3.5) .attr("cx", function(d) { return x(d.check_time); }) .attr("cy", function(d) { return y(d.duration); }) @@ -81,16 +107,16 @@ function mirror_status(chart_id, data_url) { /* invoke the data-fetch + first draw */ var cached_data = null; d3.json(data_url, function(json) { - cached_data = jQuery.map(json['urls'], - function(url, i) { - return jQuery.map(url['logs'], - function(log, j) { - return { - url: url['url'], - duration: log['duration'], - check_time: new Date(log['check_time']) - }; - }); + cached_data = jQuery.map(json.urls, function(url, i) { + return { + url: url.url, + logs: jQuery.map(url.logs, function(log, j) { + return { + duration: log.duration, + check_time: new Date(log.check_time) + }; + }) + }; }); draw_graph(cached_data); }); diff --git a/mirrors/views.py b/mirrors/views.py index 4b9721dc..be01e919 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -267,7 +267,8 @@ class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): if isinstance(obj, MirrorUrl): data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj) cutoff = now() - DEFAULT_CUTOFF - data['logs'] = obj.logs.filter(check_time__gte=cutoff) + data['logs'] = obj.logs.filter( + check_time__gte=cutoff).order_by('check_time') return data if isinstance(obj, MirrorLog): data = dict((attr, getattr(obj, attr)) -- cgit v1.2.3-2-g168b From 923ebbb53abf1d77a2f21b76e88faa085251af78 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 10 Nov 2012 17:20:26 -0600 Subject: Re-add nice() calls for mirror status axes Signed-off-by: Dan McGee --- mirrors/static/mirror_status.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'mirrors') diff --git a/mirrors/static/mirror_status.js b/mirrors/static/mirror_status.js index d107d7d1..1c352a9f 100644 --- a/mirrors/static/mirror_status.js +++ b/mirrors/static/mirror_status.js @@ -23,11 +23,11 @@ function mirror_status(chart_id, data_url) { x.domain([ d3.min(data, function(c) { return d3.min(c.logs, function(v) { return v.check_time; }); }), d3.max(data, function(c) { return d3.max(c.logs, function(v) { return v.check_time; }); }) - ]); + ]).nice(d3.time.hour); y.domain([ d3.min(data, function(c) { return d3.min(c.logs, function(v) { return v.duration; }); }), d3.max(data, function(c) { return d3.max(c.logs, function(v) { return v.duration; }); }) - ]); + ]).nice(); /* build the axis lines... */ svg.append("g") -- cgit v1.2.3-2-g168b From e26d5722289bd2e972633891d8dac09296b0cbc4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 11 Nov 2012 14:55:37 -0600 Subject: Mirror graph tweaking after usage with real data * Clamp y-axis minimum to 0. * Don't plot `is_success == false` values. * Ensure URLs are sorted predictably. Signed-off-by: Dan McGee --- mirrors/static/mirror_status.js | 5 ++++- mirrors/utils.py | 3 ++- mirrors/views.py | 2 -- 3 files changed, 6 insertions(+), 4 deletions(-) (limited to 'mirrors') diff --git a/mirrors/static/mirror_status.js b/mirrors/static/mirror_status.js index 1c352a9f..decc8fb8 100644 --- a/mirrors/static/mirror_status.js +++ b/mirrors/static/mirror_status.js @@ -25,7 +25,7 @@ function mirror_status(chart_id, data_url) { d3.max(data, function(c) { return d3.max(c.logs, function(v) { return v.check_time; }); }) ]).nice(d3.time.hour); y.domain([ - d3.min(data, function(c) { return d3.min(c.logs, function(v) { return v.duration; }); }), + 0, d3.max(data, function(c) { return d3.max(c.logs, function(v) { return v.duration; }); }) ]).nice(); @@ -111,6 +111,9 @@ function mirror_status(chart_id, data_url) { return { url: url.url, logs: jQuery.map(url.logs, function(log, j) { + if (!log.is_success) { + return null; + } return { duration: log.duration, check_time: new Date(log.check_time) diff --git a/mirrors/utils.py b/mirrors/utils.py index ba027c99..85e4ee93 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -41,7 +41,8 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): success_count=Count('logs__duration'), last_sync=Max('logs__last_sync'), last_check=Max('logs__check_time'), - duration_avg=Avg('logs__duration')) + duration_avg=Avg('logs__duration')).order_by( + 'mirror', 'url') if mirror_ids: urls = urls.filter(mirror_id__in=mirror_ids) diff --git a/mirrors/views.py b/mirrors/views.py index be01e919..5e374b4d 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -189,8 +189,6 @@ def mirror_details_json(request, name): status_info = get_mirror_statuses(mirror_ids=[mirror.id]) data = status_info.copy() data['version'] = 3 - # include only URLs for this particular mirror - data['urls'] = [url for url in data['urls'] if url.mirror_id == mirror.id] to_json = json.dumps(data, ensure_ascii=False, cls=ExtendedMirrorStatusJSONEncoder) response = HttpResponse(to_json, mimetype='application/json') -- cgit v1.2.3-2-g168b From 99bfdda5f257107396179694b7c56aad8e5e6701 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 11 Nov 2012 16:40:21 -0600 Subject: Add title to mirror status data points Signed-off-by: Dan McGee --- mirrors/static/mirror_status.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/static/mirror_status.js b/mirrors/static/mirror_status.js index decc8fb8..8ec85c40 100644 --- a/mirrors/static/mirror_status.js +++ b/mirrors/static/mirror_status.js @@ -81,7 +81,9 @@ function mirror_status(chart_id, data_url) { .attr("r", 3.5) .attr("cx", function(d) { return x(d.check_time); }) .attr("cy", function(d) { return y(d.duration); }) - .style("fill", function(d) { return color(d.url); }); + .style("fill", function(d) { return color(d.url); }) + .append("title") + .text(function(d) { return d.url + "\n" + d.duration.toFixed(3) + " secs\n" + d.check_time.toUTCString(); }); /* add a legend for good measure */ var legend = svg.selectAll(".legend") -- cgit v1.2.3-2-g168b From 55d2d3ae51e1d0aa0927d682b45a3500588ed07b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 12 Nov 2012 09:41:28 -0600 Subject: Add get_latest_by to MirrorLog Meta class Signed-off-by: Dan McGee --- mirrors/models.py | 1 + 1 file changed, 1 insertion(+) (limited to 'mirrors') diff --git a/mirrors/models.py b/mirrors/models.py index 06b483d5..384668b8 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -131,5 +131,6 @@ class MirrorLog(models.Model): class Meta: verbose_name = 'mirror check log' + get_latest_by = 'check_time' # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From 92837c93acc66056391dd0b98515b89f8fc49691 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 12 Nov 2012 21:37:08 -0600 Subject: Prefetch the available protocols on the mirror overview page Otherwise we are doing one query per mirror, which at this point is over 100 separate queries. Signed-off-by: Dan McGee --- mirrors/models.py | 5 ----- mirrors/views.py | 8 ++++++++ 2 files changed, 8 insertions(+), 5 deletions(-) (limited to 'mirrors') diff --git a/mirrors/models.py b/mirrors/models.py index 384668b8..0179d5bf 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -33,11 +33,6 @@ class Mirror(models.Model): def __unicode__(self): return self.name - def supported_protocols(self): - protocols = MirrorProtocol.objects.filter( - urls__mirror=self).order_by('protocol').distinct() - return sorted(protocols) - def downstream(self): return Mirror.objects.filter(upstream=self).order_by('name') diff --git a/mirrors/views.py b/mirrors/views.py index 5e374b4d..2e1e83b6 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -159,8 +159,16 @@ def find_mirrors_simple(request, protocol): def mirrors(request): mirror_list = Mirror.objects.select_related().order_by('tier', 'country') + protos = MirrorUrl.objects.values_list( + 'mirror_id', 'protocol__protocol').order_by( + 'mirror__id', 'protocol__protocol') if not request.user.is_authenticated(): mirror_list = mirror_list.filter(public=True, active=True) + protos = protos.filter(mirror__public=True, mirror__active=True) + protos = dict((k, list(v)) for k, v in groupby(protos, key=itemgetter(0))) + for mirror in mirror_list: + items = protos.get(mirror.id, []) + mirror.protocols = [item[1] for item in items] return render(request, 'mirrors/mirrors.html', {'mirror_list': mirror_list}) -- cgit v1.2.3-2-g168b From a2cfa7edbbed8edb1ad4d3391c6edb055c13de1b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 16 Nov 2012 15:39:56 -0600 Subject: Optimize mirror status data fetching Now that we have as many mirror URLs as we do, we can do a better job fetching and aggregating this data. The prior method resulted in a rather unwieldy query being pushed down to the database with a horrendously long GROUP BY clause. Instead of trying to group by everything at once so we can retrieve mirror URL info at the same time, separate the two queries- one for getting URL performance data, one for the qualitative data. The impetus behind fixing this is the PostgreSQL slow query log in production; this currently shows up the most of any queries we run in the system. Signed-off-by: Dan McGee --- mirrors/utils.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index 85e4ee93..07a7138f 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -33,23 +33,26 @@ def annotate_url(url, delays): @cache_function(123) def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): cutoff_time = now() - cutoff - # I swear, this actually has decent performance... - urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( + url_data = MirrorUrl.objects.values('id', 'mirror_id').filter( mirror__active=True, mirror__public=True, logs__check_time__gte=cutoff_time).annotate( check_count=Count('logs'), success_count=Count('logs__duration'), last_sync=Max('logs__last_sync'), last_check=Max('logs__check_time'), - duration_avg=Avg('logs__duration')).order_by( - 'mirror', 'url') - - if mirror_ids: - urls = urls.filter(mirror_id__in=mirror_ids) + duration_avg=Avg('logs__duration')) vendor = database_vendor(MirrorUrl) if vendor != 'sqlite': - urls = urls.annotate(duration_stddev=StdDev('logs__duration')) + url_data = url_data.annotate(duration_stddev=StdDev('logs__duration')) + + urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( + mirror__active=True, mirror__public=True, + logs__check_time__gte=cutoff_time).order_by('mirror__id', 'url') + + if mirror_ids: + url_data = url_data.filter(mirror_id__in=mirror_ids) + urls = urls.filter(mirror_id__in=mirror_ids) # The Django ORM makes it really hard to get actual average delay in the # above query, so run a seperate query for it and we will process the @@ -66,6 +69,11 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): delays.setdefault(url_id, []).append(delay) if urls: + url_data = dict((item['id'], item) for item in url_data) + for url in urls: + for k, v in url_data.get(url.id, {}).items(): + if k not in ('id', 'mirror_id'): + setattr(url, k, v) last_check = max([u.last_check for u in urls]) num_checks = max([u.check_count for u in urls]) check_info = MirrorLog.objects.filter(check_time__gte=cutoff_time) -- cgit v1.2.3-2-g168b From 6dd4d54bb0adbbb0f8c2b1beaa92b7a58971cf88 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 16 Nov 2012 16:20:11 -0600 Subject: Use Python 2.7 dictionary comprehension syntax Rather than the old idiom of dict((k, v) for <> in <>). Signed-off-by: Dan McGee --- mirrors/views.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) (limited to 'mirrors') diff --git a/mirrors/views.py b/mirrors/views.py index 2e1e83b6..d0ce0a97 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -94,7 +94,7 @@ def default_protocol_filter(original_urls): def status_filter(original_urls): status_info = get_mirror_statuses() - scores = dict((u.id, u.score) for u in status_info['urls']) + scores = {u.id: u.score for u in status_info['urls']} urls = [] for u in original_urls: u.score = scores.get(u.id, None) @@ -165,7 +165,7 @@ def mirrors(request): if not request.user.is_authenticated(): mirror_list = mirror_list.filter(public=True, active=True) protos = protos.filter(mirror__public=True, mirror__active=True) - protos = dict((k, list(v)) for k, v in groupby(protos, key=itemgetter(0))) + protos = {k: list(v) for k, v in groupby(protos, key=itemgetter(0))} for mirror in mirror_list: items = protos.get(mirror.id, []) mirror.protocols = [item[1] for item in items] @@ -253,8 +253,7 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder): # mainly for queryset serialization return list(obj) if isinstance(obj, MirrorUrl): - data = dict((attr, getattr(obj, attr)) - for attr in self.url_attributes) + data = {attr: getattr(obj, attr) for attr in self.url_attributes} # get any override on the country attribute first country = obj.real_country data['country'] = unicode(country.name) @@ -277,9 +276,7 @@ class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): check_time__gte=cutoff).order_by('check_time') return data if isinstance(obj, MirrorLog): - data = dict((attr, getattr(obj, attr)) - for attr in self.log_attributes) - return data + return {attr: getattr(obj, attr) for attr in self.log_attributes} return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) -- cgit v1.2.3-2-g168b From f0f6f7235a62186c1cae9c79036dde5d8821373d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 19 Nov 2012 08:10:21 -0600 Subject: Fix mirror URL duplication in status view We need to ensure we don't duplicate URLs in the status view, so add a distinct() call back in to the queryset when it was inadvertently dropped in commit a2cfa7edbb. This negates a lot of the performance gains we had, unfortunately, so it looks like a nested subquery might be more efficient. Disappointing the planner can't do this for us. Signed-off-by: Dan McGee --- mirrors/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index 07a7138f..a62c7f05 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -48,7 +48,8 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( mirror__active=True, mirror__public=True, - logs__check_time__gte=cutoff_time).order_by('mirror__id', 'url') + logs__check_time__gte=cutoff_time).distinct().order_by( + 'mirror__id', 'url') if mirror_ids: url_data = url_data.filter(mirror_id__in=mirror_ids) -- cgit v1.2.3-2-g168b From 2b68963ad1049f4b0198bed6da16e385aedc119e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 27 Dec 2012 16:37:49 -0600 Subject: Ensure mirror protocols are distinct Signed-off-by: Dan McGee --- mirrors/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/views.py b/mirrors/views.py index d0ce0a97..22da631a 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -161,7 +161,7 @@ def mirrors(request): mirror_list = Mirror.objects.select_related().order_by('tier', 'country') protos = MirrorUrl.objects.values_list( 'mirror_id', 'protocol__protocol').order_by( - 'mirror__id', 'protocol__protocol') + 'mirror__id', 'protocol__protocol').distinct() if not request.user.is_authenticated(): mirror_list = mirror_list.filter(public=True, active=True) protos = protos.filter(mirror__public=True, mirror__active=True) -- cgit v1.2.3-2-g168b From bec73c7a37c07821f145dbcf11435d4f2b94a149 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 12 Jan 2013 15:56:36 -0600 Subject: Round two of mirror status query improvements This seems to generate much more performant queries at the database level than what we were previously doing, and also doesn't show duplicate rows. Signed-off-by: Dan McGee --- mirrors/utils.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index a62c7f05..1d560021 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -33,9 +33,16 @@ def annotate_url(url, delays): @cache_function(123) def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): cutoff_time = now() - cutoff - url_data = MirrorUrl.objects.values('id', 'mirror_id').filter( + + valid_urls = MirrorUrl.objects.filter( mirror__active=True, mirror__public=True, - logs__check_time__gte=cutoff_time).annotate( + logs__check_time__gte=cutoff_time).distinct() + + if mirror_ids: + valid_urls = valid_urls.filter(mirror_id__in=mirror_ids) + + url_data = MirrorUrl.objects.values('id', 'mirror_id').filter( + id__in=valid_urls, logs__check_time__gte=cutoff_time).annotate( check_count=Count('logs'), success_count=Count('logs__duration'), last_sync=Max('logs__last_sync'), @@ -47,13 +54,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): url_data = url_data.annotate(duration_stddev=StdDev('logs__duration')) urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( - mirror__active=True, mirror__public=True, - logs__check_time__gte=cutoff_time).distinct().order_by( - 'mirror__id', 'url') - - if mirror_ids: - url_data = url_data.filter(mirror_id__in=mirror_ids) - urls = urls.filter(mirror_id__in=mirror_ids) + id__in=valid_urls).order_by('mirror__id', 'url') # The Django ORM makes it really hard to get actual average delay in the # above query, so run a seperate query for it and we will process the -- cgit v1.2.3-2-g168b From 66850026ca934e5a09238e9033c541cdc5085a42 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 13 Jan 2013 22:34:33 -0600 Subject: Use content_type and not mimetype on HttpResponse() Bug #16519 in Django deprecates mimetype, so update our code accordingly. Signed-off-by: Dan McGee --- mirrors/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'mirrors') diff --git a/mirrors/views.py b/mirrors/views.py index 22da631a..d3867802 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -199,7 +199,7 @@ def mirror_details_json(request, name): data['version'] = 3 to_json = json.dumps(data, ensure_ascii=False, cls=ExtendedMirrorStatusJSONEncoder) - response = HttpResponse(to_json, mimetype='application/json') + response = HttpResponse(to_json, content_type='application/json') return response @@ -285,7 +285,7 @@ def status_json(request): data = status_info.copy() data['version'] = 3 to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder) - response = HttpResponse(to_json, mimetype='application/json') + response = HttpResponse(to_json, content_type='application/json') return response # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From 5b8b6991b0b9f8174f153fe4b46376451b222cc1 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 14 Jan 2013 00:34:49 -0600 Subject: Add migration to move country data down to the URL level Rather than have the weird indirection we need now to find the right country for URLs, just always store it on the URL. Signed-off-by: Dan McGee --- .../migrations/0019_move_country_data_to_url.py | 74 ++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 mirrors/migrations/0019_move_country_data_to_url.py (limited to 'mirrors') diff --git a/mirrors/migrations/0019_move_country_data_to_url.py b/mirrors/migrations/0019_move_country_data_to_url.py new file mode 100644 index 00000000..81b7bb3e --- /dev/null +++ b/mirrors/migrations/0019_move_country_data_to_url.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import DataMigration +from django.db import models + +class Migration(DataMigration): + + def forwards(self, orm): + for url in orm.MirrorUrl.objects.select_related('mirror').all(): + # set the country field on the URL if we have one, + # and it isn't already set to anything. + if url.country or not url.mirror.country: + continue + orm.MirrorUrl.objects.filter(pk=url.pk).update( + country=url.mirror.country) + + def backwards(self, orm): + pass + + models = { + 'mirrors.mirror': { + 'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + 'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) + }, + 'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) + }, + 'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] + symmetrical = True -- cgit v1.2.3-2-g168b From 6f0ae6746baea657ee6d7c21ac0813a04f825443 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 14 Jan 2013 01:00:11 -0600 Subject: Drop country column from mirror table We now always look for this information at the URL level, not the mirror level. This simplifies quite a bit of code in and around the mirror views. Signed-off-by: Dan McGee --- mirrors/admin.py | 4 +- .../0020_auto__del_field_mirror_country.py | 70 ++++++++++++++++++++++ mirrors/models.py | 7 +-- mirrors/utils.py | 7 +-- mirrors/views.py | 15 ++--- 5 files changed, 81 insertions(+), 22 deletions(-) create mode 100644 mirrors/migrations/0020_auto__del_field_mirror_country.py (limited to 'mirrors') diff --git a/mirrors/admin.py b/mirrors/admin.py index 65fff368..eaa38391 100644 --- a/mirrors/admin.py +++ b/mirrors/admin.py @@ -62,9 +62,9 @@ class MirrorAdminForm(forms.ModelForm): class MirrorAdmin(admin.ModelAdmin): form = MirrorAdminForm - list_display = ('name', 'tier', 'country', 'active', 'public', + list_display = ('name', 'tier', 'active', 'public', 'isos', 'admin_email', 'alternate_email') - list_filter = ('tier', 'active', 'public', 'country') + list_filter = ('tier', 'active', 'public') search_fields = ('name', 'admin_email', 'alternate_email') inlines = [ MirrorUrlInlineAdmin, diff --git a/mirrors/migrations/0020_auto__del_field_mirror_country.py b/mirrors/migrations/0020_auto__del_field_mirror_country.py new file mode 100644 index 00000000..c2220a50 --- /dev/null +++ b/mirrors/migrations/0020_auto__del_field_mirror_country.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.delete_column('mirrors_mirror', 'country') + + + def backwards(self, orm): + db.add_column('mirrors_mirror', 'country', + self.gf('django_countries.fields.CountryField')(blank=True, default='', max_length=2, db_index=True), + keep_default=False) + + + models = { + 'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + 'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) + }, + 'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) + }, + 'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index 0179d5bf..ca421d13 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -17,7 +17,6 @@ class Mirror(models.Model): name = models.CharField(max_length=255, unique=True) tier = models.SmallIntegerField(default=2, choices=TIER_CHOICES) upstream = models.ForeignKey('self', null=True, on_delete=models.SET_NULL) - country = CountryField(blank=True, db_index=True) admin_email = models.EmailField(max_length=255, blank=True) alternate_email = models.EmailField(max_length=255, blank=True) public = models.BooleanField(default=True) @@ -28,7 +27,7 @@ class Mirror(models.Model): notes = models.TextField(blank=True) class Meta: - ordering = ('country', 'name') + ordering = ('name',) def __unicode__(self): return self.name @@ -75,10 +74,6 @@ class MirrorUrl(models.Model): def hostname(self): return urlparse(self.url).hostname - @property - def real_country(self): - return self.country or self.mirror.country - def clean(self): try: # Auto-map the protocol field by looking at the URL diff --git a/mirrors/utils.py b/mirrors/utils.py index 1d560021..3ab176b3 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -115,7 +115,7 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None): is_success=False, check_time__gte=cutoff_time, url__mirror__active=True, url__mirror__public=True).values( 'url__url', 'url__country', 'url__protocol__protocol', - 'url__mirror__country', 'url__mirror__tier', 'error').annotate( + 'url__mirror__tier', 'error').annotate( error_count=Count('error'), last_occurred=Max('check_time') ).order_by('-last_occurred', '-error_count') @@ -124,8 +124,7 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None): errors = list(errors) for err in errors: - ctry_code = err['url__country'] or err['url__mirror__country'] - err['country'] = Country(ctry_code) + err['country'] = Country(err['url__country']) return errors @@ -152,7 +151,7 @@ def get_mirror_url_for_download(cutoff=DEFAULT_CUTOFF): mirror_urls = MirrorUrl.objects.filter( mirror__public=True, mirror__active=True, protocol__default=True) # look first for a country-agnostic URL, then fall back to any HTTP URL - filtered_urls = mirror_urls.filter(mirror__country='')[:1] + filtered_urls = mirror_urls.filter(country='')[:1] if not filtered_urls: filtered_urls = mirror_urls[:1] if not filtered_urls: diff --git a/mirrors/views.py b/mirrors/views.py index d3867802..545e3557 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -42,9 +42,6 @@ class MirrorlistForm(forms.Form): def get_countries(self): country_codes = set() - country_codes.update(Mirror.objects.filter(active=True).exclude( - country='').values_list( - 'country', flat=True).order_by().distinct()) country_codes.update(MirrorUrl.objects.filter( mirror__active=True).exclude(country='').values_list( 'country', flat=True).order_by().distinct()) @@ -81,7 +78,7 @@ def generate_mirrorlist(request): def default_protocol_filter(original_urls): - key_func = attrgetter('real_country') + key_func = attrgetter('country') sorted_urls = sorted(original_urls, key=key_func) urls = [] for _, group in groupby(sorted_urls, key=key_func): @@ -119,8 +116,7 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, protocol__in=protocols, mirror__public=True, mirror__active=True) if countries and 'all' not in countries: - qset = qset.filter(Q(country__in=countries) | - Q(mirror__country__in=countries)) + qset = qset.filter(country__in=countries) ip_version = Q() if ipv4_supported: @@ -135,7 +131,7 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, urls = qset if not use_status: - sort_key = attrgetter('real_country.name', 'mirror.name', 'url') + sort_key = attrgetter('country.name', 'mirror.name', 'url') urls = sorted(urls, key=sort_key) template = 'mirrors/mirrorlist.txt' else: @@ -158,7 +154,7 @@ def find_mirrors_simple(request, protocol): def mirrors(request): - mirror_list = Mirror.objects.select_related().order_by('tier', 'country') + mirror_list = Mirror.objects.select_related().order_by('tier', 'name') protos = MirrorUrl.objects.values_list( 'mirror_id', 'protocol__protocol').order_by( 'mirror__id', 'protocol__protocol').distinct() @@ -254,8 +250,7 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder): return list(obj) if isinstance(obj, MirrorUrl): data = {attr: getattr(obj, attr) for attr in self.url_attributes} - # get any override on the country attribute first - country = obj.real_country + country = obj.country data['country'] = unicode(country.name) data['country_code'] = country.code return data -- cgit v1.2.3-2-g168b From ff6db38f1dc6ed1eb53454a7e16615ec1ad76d7a Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 14 Jan 2013 01:27:34 -0600 Subject: Ensure URLs without check data work on mirror details page Less noticeable in production as the templates don't show '@@@INVALID@@@' there, but we were trying to access attributes that don't actually exist on certain mirror objects. Signed-off-by: Dan McGee --- mirrors/views.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'mirrors') diff --git a/mirrors/views.py b/mirrors/views.py index 545e3557..30df5472 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -176,13 +176,17 @@ def mirror_details(request, name): raise Http404 status_info = get_mirror_statuses(mirror_ids=[mirror.id]) - checked_urls = [url for url in status_info['urls'] \ - if url.mirror_id == mirror.id] - all_urls = mirror.urls.select_related('protocol') - # get each item from checked_urls and supplement with anything in all_urls - # if it wasn't there - all_urls = set(checked_urls).union(all_urls) - all_urls = sorted(all_urls, key=attrgetter('url')) + checked_urls = {url for url in status_info['urls'] \ + if url.mirror_id == mirror.id} + all_urls = set(mirror.urls.select_related('protocol')) + # Add dummy data for URLs that we haven't checked recently + other_urls = all_urls.difference(checked_urls) + print other_urls + for url in other_urls: + for attr in ('last_sync', 'completion_pct', 'delay', 'duration_avg', + 'duration_stddev', 'score'): + setattr(url, attr, None) + all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url')) return render(request, 'mirrors/mirror_details.html', {'mirror': mirror, 'urls': all_urls}) -- cgit v1.2.3-2-g168b From 0f6a0a1cd0011c8ad137a4b27d0b39a7e1129fb7 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 14 Jan 2013 08:54:33 -0600 Subject: Support mirror status JSON by tier Just as we do for the normal status HTML view. Signed-off-by: Dan McGee --- mirrors/urls.py | 1 + mirrors/views.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/urls.py b/mirrors/urls.py index 857e99e2..4e929410 100644 --- a/mirrors/urls.py +++ b/mirrors/urls.py @@ -5,6 +5,7 @@ urlpatterns = patterns('mirrors.views', (r'^status/$', 'status', {}, 'mirror-status'), (r'^status/json/$', 'status_json', {}, 'mirror-status-json'), (r'^status/tier/(?P\d+)/$', 'status', {}, 'mirror-status-tier'), + (r'^status/tier/(?P\d+)/json/$', 'status_json', {}, 'mirror-status-tier-json'), (r'^(?P[\.\-\w]+)/$', 'mirror_details'), (r'^(?P[\.\-\w]+)/json/$', 'mirror_details_json'), ) diff --git a/mirrors/views.py b/mirrors/views.py index 30df5472..c0ed6670 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -279,9 +279,15 @@ class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) -def status_json(request): +def status_json(request, tier=None): + if tier is not None: + tier = int(tier) + if tier not in [t[0] for t in Mirror.TIER_CHOICES]: + raise Http404 status_info = get_mirror_statuses() data = status_info.copy() + if tier is not None: + data['urls'] = [url for url in data['urls'] if url.mirror.tier == tier] data['version'] = 3 to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder) response = HttpResponse(to_json, content_type='application/json') -- cgit v1.2.3-2-g168b From 1f9aef78f39c90191eddf2233c278086a15052de Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 16 Jan 2013 00:18:26 -0600 Subject: Remove debugging print statement Signed-off-by: Dan McGee --- mirrors/views.py | 1 - 1 file changed, 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/views.py b/mirrors/views.py index c0ed6670..56397633 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -181,7 +181,6 @@ def mirror_details(request, name): all_urls = set(mirror.urls.select_related('protocol')) # Add dummy data for URLs that we haven't checked recently other_urls = all_urls.difference(checked_urls) - print other_urls for url in other_urls: for attr in ('last_sync', 'completion_pct', 'delay', 'duration_avg', 'duration_stddev', 'score'): -- cgit v1.2.3-2-g168b From dd8e94f69783365160bcbfda61a9bebea5a71324 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 20 Jan 2013 14:53:28 -0600 Subject: Lengthen the mirror rsync IP address field Make it long enough to support a full-form IPv6 address with a subnet. Signed-off-by: Dan McGee --- .../0021_auto__chg_field_mirrorrsync_ip.py | 66 ++++++++++++++++++++++ mirrors/models.py | 5 +- 2 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 mirrors/migrations/0021_auto__chg_field_mirrorrsync_ip.py (limited to 'mirrors') diff --git a/mirrors/migrations/0021_auto__chg_field_mirrorrsync_ip.py b/mirrors/migrations/0021_auto__chg_field_mirrorrsync_ip.py new file mode 100644 index 00000000..bbf14bb0 --- /dev/null +++ b/mirrors/migrations/0021_auto__chg_field_mirrorrsync_ip.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.alter_column(u'mirrors_mirrorrsync', 'ip', self.gf('django.db.models.fields.CharField')(max_length=44)) + + def backwards(self, orm): + db.alter_column(u'mirrors_mirrorrsync', 'ip', self.gf('django.db.models.fields.CharField')(max_length=24)) + + models = { + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index ca421d13..ec4a044d 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -98,11 +98,12 @@ class MirrorUrl(models.Model): class MirrorRsync(models.Model): - ip = models.CharField("IP", max_length=24) + # max length is 40 chars for full-form IPv6 addr + subnet + ip = models.CharField("IP", max_length=44) mirror = models.ForeignKey(Mirror, related_name="rsync_ips") def __unicode__(self): - return "%s" % (self.ip) + return self.ip class Meta: verbose_name = 'mirror rsync IP' -- cgit v1.2.3-2-g168b From 5566d43a7734f6bb2f48d5d511351da12ddc5cc1 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 9 Feb 2013 16:43:40 -0600 Subject: Use 'update_fields' model.save() kwarg This was added in Django 1.5 and allows saving only a subset of a model's fields. It makes sense in a few cases to utilize it. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorresolv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorresolv.py b/mirrors/management/commands/mirrorresolv.py index 0370f8ed..a6c2523e 100644 --- a/mirrors/management/commands/mirrorresolv.py +++ b/mirrors/management/commands/mirrorresolv.py @@ -53,7 +53,7 @@ def resolve_mirrors(): newvals = (mirrorurl.has_ipv4, mirrorurl.has_ipv6) if newvals != oldvals: logger.debug("values changed for %s", mirrorurl) - mirrorurl.save(force_update=True) + mirrorurl.save(update_fields=('has_ipv4', 'has_ipv6')) except socket.error, e: logger.warn("error resolving %s: %s", mirrorurl.hostname, e) -- cgit v1.2.3-2-g168b From 10af269ed21b00cb3da9b380f72016fefae52a6d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 4 Mar 2013 18:11:28 -0600 Subject: Add HTTPS mirror protocol fixture For now, it is not included in the default selection, but we have a few existing mirrors that do support it. Signed-off-by: Dan McGee --- mirrors/fixtures/mirrorprotocols.json | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'mirrors') diff --git a/mirrors/fixtures/mirrorprotocols.json b/mirrors/fixtures/mirrorprotocols.json index 72ed1a7f..8822ef8e 100644 --- a/mirrors/fixtures/mirrorprotocols.json +++ b/mirrors/fixtures/mirrorprotocols.json @@ -7,7 +7,7 @@ "default": true, "protocol": "http" } - }, + }, { "pk": 2, "model": "mirrors.mirrorprotocol", @@ -16,7 +16,7 @@ "default": false, "protocol": "ftp" } - }, + }, { "pk": 3, "model": "mirrors.mirrorprotocol", @@ -25,5 +25,14 @@ "default": false, "protocol": "rsync" } + }, + { + "pk": 5, + "model": "mirrors.mirrorprotocol", + "fields": { + "is_download": true, + "default": false, + "protocol": "https" + } } ] -- cgit v1.2.3-2-g168b From d158ce71e4ec489ee3ec1a73c41c9b9dc8d34a23 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 6 Mar 2013 19:40:24 -0600 Subject: Add mirror check locations model Signed-off-by: Dan McGee --- mirrors/migrations/0022_auto__add_checklocation.py | 83 ++++++++++++++++++++++ mirrors/models.py | 23 +++++- 2 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 mirrors/migrations/0022_auto__add_checklocation.py (limited to 'mirrors') diff --git a/mirrors/migrations/0022_auto__add_checklocation.py b/mirrors/migrations/0022_auto__add_checklocation.py new file mode 100644 index 00000000..896b2dab --- /dev/null +++ b/mirrors/migrations/0022_auto__add_checklocation.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.create_table(u'mirrors_checklocation', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('hostname', self.gf('django.db.models.fields.CharField')(max_length=255)), + ('source_ip', self.gf('django.db.models.fields.GenericIPAddressField')(unique=True, max_length=39)), + ('country', self.gf('django_countries.fields.CountryField')(max_length=2)), + ('created', self.gf('django.db.models.fields.DateTimeField')()), + )) + db.send_create_signal(u'mirrors', ['CheckLocation']) + + + def backwards(self, orm): + db.delete_table(u'mirrors_checklocation') + + + models = { + u'mirrors.checklocation': { + 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'}, + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'}) + }, + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index ec4a044d..c7a0a93f 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -1,10 +1,13 @@ import socket from urlparse import urlparse -from django.db import models from django.core.exceptions import ValidationError +from django.db import models +from django.db.models.signals import pre_save from django_countries import CountryField +from main.utils import set_created_field + class Mirror(models.Model): TIER_CHOICES = ( @@ -109,6 +112,20 @@ class MirrorRsync(models.Model): verbose_name = 'mirror rsync IP' +class CheckLocation(models.Model): + hostname = models.CharField(max_length=255) + source_ip = models.GenericIPAddressField(verbose_name='source IP', + unpack_ipv4=True, unique=True) + country = CountryField() + created = models.DateTimeField(editable=False) + + class Meta: + ordering = ('hostname', 'source_ip') + + def __unicode__(self): + return self.hostname + + class MirrorLog(models.Model): url = models.ForeignKey(MirrorUrl, related_name="logs") check_time = models.DateTimeField(db_index=True) @@ -124,4 +141,8 @@ class MirrorLog(models.Model): verbose_name = 'mirror check log' get_latest_by = 'check_time' + +pre_save.connect(set_created_field, sender=CheckLocation, + dispatch_uid="mirrors.models") + # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From 1dbf311774f7894cac870517558d8baee8681f0d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 6 Mar 2013 19:45:52 -0600 Subject: Add 'created' field to more mirror models We have been better about doing this to most of our models, but the ones here didn't have a created field. Add it where appropriate and set a reasonably old default value. Signed-off-by: Dan McGee --- ...created__add_field_mirrorrsync_created__add_.py | 97 ++++++++++++++++++++++ mirrors/models.py | 9 +- 2 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 mirrors/migrations/0023_auto__add_field_mirrorurl_created__add_field_mirrorrsync_created__add_.py (limited to 'mirrors') diff --git a/mirrors/migrations/0023_auto__add_field_mirrorurl_created__add_field_mirrorrsync_created__add_.py b/mirrors/migrations/0023_auto__add_field_mirrorurl_created__add_field_mirrorrsync_created__add_.py new file mode 100644 index 00000000..1a1f48a0 --- /dev/null +++ b/mirrors/migrations/0023_auto__add_field_mirrorurl_created__add_field_mirrorrsync_created__add_.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models +from pytz import utc + + +class Migration(SchemaMigration): + + def forwards(self, orm): + default = datetime.datetime(2000, 1, 1, 0, 0).replace(tzinfo=utc) + db.add_column(u'mirrors_mirrorurl', 'created', + self.gf('django.db.models.fields.DateTimeField')(default=default), + keep_default=False) + db.add_column(u'mirrors_mirrorrsync', 'created', + self.gf('django.db.models.fields.DateTimeField')(default=default), + keep_default=False) + db.add_column(u'mirrors_mirrorprotocol', 'created', + self.gf('django.db.models.fields.DateTimeField')(default=default), + keep_default=False) + db.add_column(u'mirrors_mirror', 'created', + self.gf('django.db.models.fields.DateTimeField')(default=default), + keep_default=False) + + + def backwards(self, orm): + db.delete_column(u'mirrors_mirrorurl', 'created') + db.delete_column(u'mirrors_mirrorrsync', 'created') + db.delete_column(u'mirrors_mirrorprotocol', 'created') + db.delete_column(u'mirrors_mirror', 'created') + + + models = { + u'mirrors.checklocation': { + 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'}, + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'}) + }, + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index c7a0a93f..c205fef2 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -28,6 +28,7 @@ class Mirror(models.Model): rsync_user = models.CharField(max_length=50, blank=True, default='') rsync_password = models.CharField(max_length=50, blank=True, default='') notes = models.TextField(blank=True) + created = models.DateTimeField(editable=False) class Meta: ordering = ('name',) @@ -48,6 +49,7 @@ class MirrorProtocol(models.Model): help_text="Is protocol useful for end-users, e.g. FTP/HTTP") default = models.BooleanField(default=True, help_text="Included by default when building mirror list?") + created = models.DateTimeField(editable=False) def __unicode__(self): return self.protocol @@ -66,6 +68,7 @@ class MirrorUrl(models.Model): editable=False) has_ipv6 = models.BooleanField("IPv6 capable", default=False, editable=False) + created = models.DateTimeField(editable=False) def address_families(self): hostname = urlparse(self.url).hostname @@ -104,6 +107,7 @@ class MirrorRsync(models.Model): # max length is 40 chars for full-form IPv6 addr + subnet ip = models.CharField("IP", max_length=44) mirror = models.ForeignKey(Mirror, related_name="rsync_ips") + created = models.DateTimeField(editable=False) def __unicode__(self): return self.ip @@ -142,7 +146,8 @@ class MirrorLog(models.Model): get_latest_by = 'check_time' -pre_save.connect(set_created_field, sender=CheckLocation, - dispatch_uid="mirrors.models") +for model in (Mirror, MirrorProtocol, MirrorUrl, MirrorRsync, CheckLocation): + pre_save.connect(set_created_field, sender=model, + dispatch_uid="mirrors.models") # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From 71259ab4c27ca6f00e09e813c7d9c6e8e24d59b4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 6 Mar 2013 19:53:11 -0600 Subject: Add mirror CheckLocationAdmin Signed-off-by: Dan McGee --- mirrors/admin.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/admin.py b/mirrors/admin.py index eaa38391..d6ea3950 100644 --- a/mirrors/admin.py +++ b/mirrors/admin.py @@ -4,7 +4,9 @@ from urlparse import urlparse, urlunsplit from django import forms from django.contrib import admin -from .models import Mirror, MirrorProtocol, MirrorUrl, MirrorRsync +from .models import (Mirror, MirrorProtocol, MirrorUrl, MirrorRsync, + CheckLocation) + class MirrorUrlForm(forms.ModelForm): class Meta: @@ -26,12 +28,14 @@ class MirrorUrlForm(forms.ModelForm): url = urlunsplit((url_parts.scheme, url_parts.netloc, path, '', '')) return url + class MirrorUrlInlineAdmin(admin.TabularInline): model = MirrorUrl form = MirrorUrlForm readonly_fields = ('protocol', 'has_ipv4', 'has_ipv6') extra = 3 + # ripped off from django.forms.fields, adding netmask ability IPV4NM_RE = re.compile(r'^(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}(/(\d|[1-2]\d|3[0-2])){0,1}$') @@ -43,16 +47,19 @@ class IPAddressNetmaskField(forms.fields.RegexField): def __init__(self, *args, **kwargs): super(IPAddressNetmaskField, self).__init__(IPV4NM_RE, *args, **kwargs) + class MirrorRsyncForm(forms.ModelForm): class Meta: model = MirrorRsync ip = IPAddressNetmaskField(label='IP') + class MirrorRsyncInlineAdmin(admin.TabularInline): model = MirrorRsync form = MirrorRsyncForm extra = 2 + class MirrorAdminForm(forms.ModelForm): class Meta: model = Mirror @@ -60,6 +67,7 @@ class MirrorAdminForm(forms.ModelForm): queryset=Mirror.objects.filter(tier__gte=0, tier__lte=1), required=False) + class MirrorAdmin(admin.ModelAdmin): form = MirrorAdminForm list_display = ('name', 'tier', 'active', 'public', @@ -71,11 +79,19 @@ class MirrorAdmin(admin.ModelAdmin): MirrorRsyncInlineAdmin, ] + class MirrorProtocolAdmin(admin.ModelAdmin): list_display = ('protocol', 'is_download', 'default') list_filter = ('is_download', 'default') + +class CheckLocationAdmin(admin.ModelAdmin): + list_display = ('hostname', 'source_ip', 'country', 'created') + search_fields = ('hostname', 'source_ip') + + admin.site.register(Mirror, MirrorAdmin) admin.site.register(MirrorProtocol, MirrorProtocolAdmin) +admin.site.register(CheckLocation, CheckLocationAdmin) # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From 3e0209f5e8ee197034b6c1f705af515d8154801b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 6 Mar 2013 19:56:48 -0600 Subject: Revert "mirrorcheck: Don't use bulk_create on sqlite3" This reverts commit 3c4ceb16. We don't need this anymore as bulk_create gets automatic batching now on sqlite3 so it is safe to use. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index e09ea680..2116ab29 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -31,7 +31,6 @@ from django.core.management.base import NoArgsCommand from django.db import transaction from django.utils.timezone import now -from main.utils import database_vendor from mirrors.models import MirrorUrl, MirrorLog logging.basicConfig( @@ -208,11 +207,7 @@ class MirrorCheckPool(object): logger.debug("joining on all threads") self.tasks.join() logger.debug("processing %d log entries", len(self.logs)) - if database_vendor(MirrorLog, mode='write') == 'sqlite': - for log in self.logs: - log.save(force_insert=True) - else: - MirrorLog.objects.bulk_create(self.logs) + MirrorLog.objects.bulk_create(self.logs) logger.debug("log entries saved") # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From d9dbe4fb1ebbf4c5d26b151509acc4ce30654b8d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 6 Mar 2013 20:43:09 -0600 Subject: Add family property to mirror check location Signed-off-by: Dan McGee --- mirrors/models.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'mirrors') diff --git a/mirrors/models.py b/mirrors/models.py index c205fef2..07ac1e6e 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -129,6 +129,13 @@ class CheckLocation(models.Model): def __unicode__(self): return self.hostname + @property + def family(self): + info = socket.getaddrinfo(self.source_ip, None, 0, 0, 0, + socket.AI_NUMERICHOST) + families = [x[0] for x in info] + return families[0] + class MirrorLog(models.Model): url = models.ForeignKey(MirrorUrl, related_name="logs") -- cgit v1.2.3-2-g168b From 9917ee3482d6fa91f779af9ad2d02097775211a4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 6 Mar 2013 20:49:38 -0600 Subject: Add location ID to mirror logs Signed-off-by: Dan McGee --- .../0024_auto__add_field_mirrorlog_location.py | 83 ++++++++++++++++++++++ mirrors/models.py | 1 + 2 files changed, 84 insertions(+) create mode 100644 mirrors/migrations/0024_auto__add_field_mirrorlog_location.py (limited to 'mirrors') diff --git a/mirrors/migrations/0024_auto__add_field_mirrorlog_location.py b/mirrors/migrations/0024_auto__add_field_mirrorlog_location.py new file mode 100644 index 00000000..acf8df17 --- /dev/null +++ b/mirrors/migrations/0024_auto__add_field_mirrorlog_location.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.add_column(u'mirrors_mirrorlog', 'location', + self.gf('django.db.models.fields.related.ForeignKey')(related_name='logs', null=True, to=orm['mirrors.CheckLocation']), + keep_default=False) + + + def backwards(self, orm): + db.delete_column(u'mirrors_mirrorlog', 'location_id') + + + models = { + u'mirrors.checklocation': { + 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'}, + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'}) + }, + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'location': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'null': 'True', 'to': u"orm['mirrors.CheckLocation']"}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index 07ac1e6e..e41f6b22 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -139,6 +139,7 @@ class CheckLocation(models.Model): class MirrorLog(models.Model): url = models.ForeignKey(MirrorUrl, related_name="logs") + location = models.ForeignKey(CheckLocation, related_name="logs", null=True) check_time = models.DateTimeField(db_index=True) last_sync = models.DateTimeField(null=True) duration = models.FloatField(null=True) -- cgit v1.2.3-2-g168b From 7c8b09b95ce5db9ddf7e895c2722bd202f5c4f54 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 6 Mar 2013 20:58:09 -0600 Subject: Teach mirrorcheck management command about check locations This adds the -l/--location argument to the command in order to pass in a check location that we are currently running from. This locks the IP address family to the one derived from the address on that location, and stores any check results tagged with a location ID. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 63 ++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 17 deletions(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 2116ab29..f133c785 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -31,7 +31,8 @@ from django.core.management.base import NoArgsCommand from django.db import transaction from django.utils.timezone import now -from mirrors.models import MirrorUrl, MirrorLog +from mirrors.models import MirrorUrl, MirrorLog, CheckLocation + logging.basicConfig( level=logging.WARNING, @@ -40,17 +41,17 @@ logging.basicConfig( stream=sys.stderr) logger = logging.getLogger() - class Command(NoArgsCommand): option_list = NoArgsCommand.option_list + ( - make_option('-t', '--timeout', dest='timeout', default='10', + make_option('-t', '--timeout', dest='timeout', type='float', default=10.0, help='Timeout value for connecting to URL'), + make_option('-l', '--location', dest='location', type='int', + help='ID of CheckLocation object to use for this run'), ) help = "Runs a check on all known mirror URLs to determine their up-to-date status." def handle_noargs(self, **options): v = int(options.get('verbosity', 0)) - timeout = int(options.get('timeout', 10)) if v == 0: logger.level = logging.ERROR elif v == 1: @@ -58,14 +59,35 @@ class Command(NoArgsCommand): elif v == 2: logger.level = logging.DEBUG + timeout = options.get('timeout') + urls = MirrorUrl.objects.select_related('protocol').filter( mirror__active=True, mirror__public=True) - pool = MirrorCheckPool(urls, timeout) + location = options.get('location', None) + if location: + location = CheckLocation.objects.get(id=location) + family = location.family + monkeypatch_getaddrinfo(family) + if family == socket.AF_INET6: + urls = urls.filter(has_ipv6=True) + elif family == socket.AF_INET: + urls = urls.filter(has_ipv4=True) + + pool = MirrorCheckPool(urls, location, timeout) pool.run() return 0 +def monkeypatch_getaddrinfo(force_family=socket.AF_INET): + '''Force the Python socket module to connect over the designated family; + e.g. socket.AF_INET or socket.AF_INET6.''' + orig = socket.getaddrinfo + def wrapper(host, port, family=0, socktype=0, proto=0, flags=0): + return orig(host, port, force_family, socktype, proto, flags) + socket.getaddrinfo = wrapper + + def parse_lastsync(log, data): '''lastsync file should be an epoch value created by us.''' try: @@ -80,10 +102,10 @@ def parse_lastsync(log, data): log.is_success = False -def check_mirror_url(mirror_url, timeout): +def check_mirror_url(mirror_url, location, timeout): url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) - log = MirrorLog(url=mirror_url, check_time=now()) + log = MirrorLog(url=mirror_url, check_time=now(), location=location) headers = {'User-Agent': 'archweb/1.0'} req = urllib2.Request(url, None, headers) try: @@ -133,17 +155,24 @@ def check_mirror_url(mirror_url, timeout): return log -def check_rsync_url(mirror_url, timeout): +def check_rsync_url(mirror_url, location, timeout): url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) - log = MirrorLog(url=mirror_url, check_time=now()) + log = MirrorLog(url=mirror_url, check_time=now(), location=location) tempdir = tempfile.mkdtemp() + ipopt = '' + if location: + if location.family == socket.AF_INET6: + ipopt = '--ipv6' + elif location.family == socket.AF_INET: + ipopt = '--ipv4' lastsync_path = os.path.join(tempdir, 'lastsync') rsync_cmd = ["rsync", "--quiet", "--contimeout=%d" % timeout, - "--timeout=%d" % timeout, url, lastsync_path] + "--timeout=%d" % timeout, ipopt, url, lastsync_path] try: with open(os.devnull, 'w') as devnull: + logger.debug("rsync cmd: %s", ' '.join(rsync_cmd)) proc = subprocess.Popen(rsync_cmd, stdout=devnull, stderr=subprocess.PIPE) start = time.time() @@ -170,15 +199,15 @@ def check_rsync_url(mirror_url, timeout): return log -def mirror_url_worker(work, output, timeout): +def mirror_url_worker(work, output, location, timeout): while True: try: url = work.get(block=False) try: if url.protocol.protocol == 'rsync': - log = check_rsync_url(url, timeout) + log = check_rsync_url(url, location, timeout) else: - log = check_mirror_url(url, timeout) + log = check_mirror_url(url, location, timeout) output.append(log) finally: work.task_done() @@ -187,15 +216,15 @@ def mirror_url_worker(work, output, timeout): class MirrorCheckPool(object): - def __init__(self, urls, timeout=10, num_threads=10): + def __init__(self, urls, location, timeout=10, num_threads=10): self.tasks = Queue() self.logs = deque() - for i in list(urls): - self.tasks.put(i) + for url in list(urls): + self.tasks.put(url) self.threads = [] for i in range(num_threads): thread = Thread(target=mirror_url_worker, - args=(self.tasks, self.logs, timeout)) + args=(self.tasks, self.logs, location, timeout)) thread.daemon = True self.threads.append(thread) -- cgit v1.2.3-2-g168b From ace95f6e53f41409568d4e4f1cf4c2a69d931e2c Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 6 Mar 2013 21:38:58 -0600 Subject: Don't add blank options to rsync command line Rsync doesn't like this so much: Unexpected remote arg: rsync://mirror.example.com/archlinux/lastsync rsync error: syntax or usage error (code 1) at main.c(1214) [sender=3.0.9] Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index f133c785..1315a013 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -169,7 +169,11 @@ def check_rsync_url(mirror_url, location, timeout): ipopt = '--ipv4' lastsync_path = os.path.join(tempdir, 'lastsync') rsync_cmd = ["rsync", "--quiet", "--contimeout=%d" % timeout, - "--timeout=%d" % timeout, ipopt, url, lastsync_path] + "--timeout=%d" % timeout] + if ipopt: + rsync_cmd.append(ipopt) + rsync_cmd.append(url) + rsync_cmd.append(lastsync_path) try: with open(os.devnull, 'w') as devnull: logger.debug("rsync cmd: %s", ' '.join(rsync_cmd)) -- cgit v1.2.3-2-g168b From 46d21e03e81e4cacc849d798052b3ffd525d638a Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 12 Mar 2013 20:18:51 -0500 Subject: Don't check FTP + IPv6 combination Very few, if any, FTP servers support connections over IPv6. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 1315a013..93b53d6b 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -41,6 +41,7 @@ logging.basicConfig( stream=sys.stderr) logger = logging.getLogger() + class Command(NoArgsCommand): option_list = NoArgsCommand.option_list + ( make_option('-t', '--timeout', dest='timeout', type='float', default=10.0, @@ -83,8 +84,10 @@ def monkeypatch_getaddrinfo(force_family=socket.AF_INET): '''Force the Python socket module to connect over the designated family; e.g. socket.AF_INET or socket.AF_INET6.''' orig = socket.getaddrinfo + def wrapper(host, port, family=0, socktype=0, proto=0, flags=0): return orig(host, port, force_family, socktype, proto, flags) + socket.getaddrinfo = wrapper @@ -103,6 +106,12 @@ def parse_lastsync(log, data): def check_mirror_url(mirror_url, location, timeout): + if location: + if location.family == socket.AF_INET6: + ipopt = '--ipv6' + elif location.family == socket.AF_INET: + ipopt = '--ipv4' + url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) log = MirrorLog(url=mirror_url, check_time=now(), location=location) @@ -210,9 +219,14 @@ def mirror_url_worker(work, output, location, timeout): try: if url.protocol.protocol == 'rsync': log = check_rsync_url(url, location, timeout) + if (url.protocol.protocol == 'ftp' and location and + location.family == socket.AF_INET6): + # IPv6 + FTP don't work; skip checking completely + log = None else: log = check_mirror_url(url, location, timeout) - output.append(log) + if log: + output.append(log) finally: work.task_done() except Empty: -- cgit v1.2.3-2-g168b From b8ee7b1ee281b45b245fb454228b8ad847c56200 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 13 Mar 2013 13:36:14 -0500 Subject: mirrorcheck: s/if/elif/ when determining what check function to run This was a silly thinko here; it caused the logs to fill up with a bunch of 'unknown url type: rsync' errors. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 93b53d6b..d6de8f22 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -219,7 +219,7 @@ def mirror_url_worker(work, output, location, timeout): try: if url.protocol.protocol == 'rsync': log = check_rsync_url(url, location, timeout) - if (url.protocol.protocol == 'ftp' and location and + elif (url.protocol.protocol == 'ftp' and location and location.family == socket.AF_INET6): # IPv6 + FTP don't work; skip checking completely log = None -- cgit v1.2.3-2-g168b From 133d16f91f3d296e188d910f609128d854e65823 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 29 Mar 2013 15:51:50 -0500 Subject: Add IP family lookup to CheckLocation model Signed-off-by: Dan McGee --- mirrors/models.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'mirrors') diff --git a/mirrors/models.py b/mirrors/models.py index e41f6b22..b0da5616 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -136,6 +136,15 @@ class CheckLocation(models.Model): families = [x[0] for x in info] return families[0] + @property + def ip_version(self): + '''Returns integer '4' or '6'.''' + if self.family == socket.AF_INET6: + return 6 + if self.family == socket.AF_INET: + return 4 + return None + class MirrorLog(models.Model): url = models.ForeignKey(MirrorUrl, related_name="logs") -- cgit v1.2.3-2-g168b From 06e1e857abfdf7f95661d337ce3c315bd51fb837 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 10 Apr 2013 21:00:17 -0500 Subject: Allow mirror rsync IPs to be IPv4/IPv6 addresses or networks This gives us a bunch more flexibility on this field, and now supports all the options that the rsync config file supports. Signed-off-by: Dan McGee --- mirrors/admin.py | 14 ---- mirrors/fields.py | 49 +++++++++++++ .../0025_auto__chg_field_mirrorrsync_ip.py | 85 ++++++++++++++++++++++ mirrors/models.py | 3 +- 4 files changed, 136 insertions(+), 15 deletions(-) create mode 100644 mirrors/fields.py create mode 100644 mirrors/migrations/0025_auto__chg_field_mirrorrsync_ip.py (limited to 'mirrors') diff --git a/mirrors/admin.py b/mirrors/admin.py index d6ea3950..9c88207d 100644 --- a/mirrors/admin.py +++ b/mirrors/admin.py @@ -1,4 +1,3 @@ -import re from urlparse import urlparse, urlunsplit from django import forms @@ -36,22 +35,9 @@ class MirrorUrlInlineAdmin(admin.TabularInline): extra = 3 -# ripped off from django.forms.fields, adding netmask ability -IPV4NM_RE = re.compile(r'^(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}(/(\d|[1-2]\d|3[0-2])){0,1}$') - -class IPAddressNetmaskField(forms.fields.RegexField): - default_error_messages = { - 'invalid': u'Enter a valid IPv4 address, possibly including netmask.', - } - - def __init__(self, *args, **kwargs): - super(IPAddressNetmaskField, self).__init__(IPV4NM_RE, *args, **kwargs) - - class MirrorRsyncForm(forms.ModelForm): class Meta: model = MirrorRsync - ip = IPAddressNetmaskField(label='IP') class MirrorRsyncInlineAdmin(admin.TabularInline): diff --git a/mirrors/fields.py b/mirrors/fields.py new file mode 100644 index 00000000..206c9d7d --- /dev/null +++ b/mirrors/fields.py @@ -0,0 +1,49 @@ +from IPy import IP + +from django import forms +from django.core import validators +from django.core.exceptions import ValidationError +from django.db import models +from south.modelsinspector import add_introspection_rules + + +class IPNetworkFormField(forms.Field): + def to_python(self, value): + if value in validators.EMPTY_VALUES: + return None + try: + value = IP(value) + except ValueError as e: + raise ValidationError(str(e)) + return value + + +class IPNetworkField(models.Field): + __metaclass__ = models.SubfieldBase + description = "IPv4 or IPv6 address or subnet" + + def __init__(self, *args, **kwargs): + kwargs['max_length'] = 44 + super(IPNetworkField, self).__init__(*args, **kwargs) + + def get_internal_type(self): + return "IPAddressField" + + def to_python(self, value): + if not value: + return None + return IP(value) + + def get_prep_value(self, value): + value = self.to_python(value) + if not value: + return None + return str(value) + + def formfield(self, **kwargs): + defaults = {'form_class': IPNetworkFormField} + defaults.update(kwargs) + return super(IPNetworkField, self).formfield(**defaults) + + +add_introspection_rules([], ["^mirrors\.fields\.IPNetworkField"]) diff --git a/mirrors/migrations/0025_auto__chg_field_mirrorrsync_ip.py b/mirrors/migrations/0025_auto__chg_field_mirrorrsync_ip.py new file mode 100644 index 00000000..b359b637 --- /dev/null +++ b/mirrors/migrations/0025_auto__chg_field_mirrorrsync_ip.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + if db.backend_name == 'postgres': + # For PostgreSQL, because it uses the 'inet' type and not a varchar + # column, we need to add an explict 'USING' cast to the SQL + # statement. We then execute the alter_column as well to ensure any + # of the other side-effects happen. + db.execute('ALTER TABLE "mirrors_mirrorrsync" ALTER COLUMN "ip" TYPE inet USING "ip"::inet') + db.alter_column(u'mirrors_mirrorrsync', 'ip', self.gf('mirrors.fields.IPNetworkField')(max_length=44)) + + def backwards(self, orm): + db.alter_column(u'mirrors_mirrorrsync', 'ip', self.gf('django.db.models.fields.CharField')(max_length=44)) + + models = { + u'mirrors.checklocation': { + 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'}, + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'}) + }, + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'location': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'null': 'True', 'to': u"orm['mirrors.CheckLocation']"}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('mirrors.fields.IPNetworkField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index b0da5616..791b0078 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -6,6 +6,7 @@ from django.db import models from django.db.models.signals import pre_save from django_countries import CountryField +from .fields import IPNetworkField from main.utils import set_created_field @@ -105,7 +106,7 @@ class MirrorUrl(models.Model): class MirrorRsync(models.Model): # max length is 40 chars for full-form IPv6 addr + subnet - ip = models.CharField("IP", max_length=44) + ip = IPNetworkField("IP") mirror = models.ForeignKey(Mirror, related_name="rsync_ips") created = models.DateTimeField(editable=False) -- cgit v1.2.3-2-g168b From 2c24ee9100a9e60fec16055d6496caeda3a1d8e2 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 13 Apr 2013 11:33:17 -0500 Subject: Calculate average URL delay in the database Rather than doing this in the Python code and needing 12,000+ rows returned from the database, we can do it in the database and get fewer than 300 rows back. If I recall correctly, the reason this was not done originally was due to our usage of MySQL and some really bad date math/overflow stuff it did when the interval between last_sync and check_time were greater than about a week. Luckily, we have switched to using a more sane database. Signed-off-by: Dan McGee --- mirrors/utils.py | 49 ++++++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 19 deletions(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index 3ab176b3..2721e20e 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -1,5 +1,6 @@ from datetime import timedelta +from django.db import connection from django.db.models import Avg, Count, Max, Min, StdDev from django.utils.timezone import now from django_countries.fields import Country @@ -10,13 +11,12 @@ from .models import MirrorLog, MirrorProtocol, MirrorUrl DEFAULT_CUTOFF = timedelta(hours=24) -def annotate_url(url, delays): +def annotate_url(url, delay): '''Given a MirrorURL object, add a few more attributes to it regarding status, including completion_pct, delay, and score.''' url.completion_pct = float(url.success_count) / url.check_count - if url.id in delays: - url_delays = delays[url.id] - url.delay = sum(url_delays, timedelta()) / len(url_delays) + if delay is not None: + url.delay = delay hours = url.delay.days * 24.0 + url.delay.seconds / 3600.0 if url.completion_pct > 0: @@ -30,6 +30,30 @@ def annotate_url(url, delays): url.score = None +def url_delays(cutoff_time, mirror_id=None): + cursor = connection.cursor() + if mirror_id is None: + sql= """ +SELECT url_id, AVG(check_time - last_sync) +FROM mirrors_mirrorlog +WHERE is_success = %s AND check_time >= %s AND last_sync IS NOT NULL +GROUP BY url_id +""" + cursor.execute(sql, [True, cutoff_time]) + else: + sql = """ +SELECT l.url_id, avg(check_time - last_sync) +FROM mirrors_mirrorlog l +JOIN mirrors_mirrorurl u ON u.id = l.url_id +WHERE is_success = %s AND check_time >= %s AND last_sync IS NOT NULL +AND mirror_id = %s +GROUP BY url_id +""" + cursor.execute(sql, [True, cutoff_time, mirror_id]) + + return {url_id: delay for url_id, delay in cursor.fetchall()} + + @cache_function(123) def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): cutoff_time = now() - cutoff @@ -55,20 +79,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( id__in=valid_urls).order_by('mirror__id', 'url') - - # The Django ORM makes it really hard to get actual average delay in the - # above query, so run a seperate query for it and we will process the - # results here. - times = MirrorLog.objects.values_list( - 'url_id', 'check_time', 'last_sync').filter( - is_success=True, last_sync__isnull=False, - check_time__gte=cutoff_time) - if mirror_ids: - times = times.filter(url__mirror_id__in=mirror_ids) - delays = {} - for url_id, check_time, last_sync in times: - delay = check_time - last_sync - delays.setdefault(url_id, []).append(delay) + delays = url_delays(cutoff_time) if urls: url_data = dict((item['id'], item) for item in url_data) @@ -97,7 +108,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): # fake the standard deviation for local testing setups if vendor == 'sqlite': setattr(url, 'duration_stddev', 0.0) - annotate_url(url, delays) + annotate_url(url, delays.get(url.id, None)) return { 'cutoff': cutoff, -- cgit v1.2.3-2-g168b From c588d1c85f86f5ee10a96bec679111c8675b703c Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 13 Apr 2013 11:38:11 -0500 Subject: Support only a single mirror ID in error/status retrieval This simplifies things and makes injecting this single mirror ID into custom SQL a whole lot easier. Signed-off-by: Dan McGee --- mirrors/utils.py | 18 +++++++++--------- mirrors/views.py | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index 2721e20e..d18dc22f 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -55,15 +55,15 @@ GROUP BY url_id @cache_function(123) -def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): +def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): cutoff_time = now() - cutoff valid_urls = MirrorUrl.objects.filter( mirror__active=True, mirror__public=True, logs__check_time__gte=cutoff_time).distinct() - if mirror_ids: - valid_urls = valid_urls.filter(mirror_id__in=mirror_ids) + if mirror_id: + valid_urls = valid_urls.filter(mirror_id=mirror_id) url_data = MirrorUrl.objects.values('id', 'mirror_id').filter( id__in=valid_urls, logs__check_time__gte=cutoff_time).annotate( @@ -79,7 +79,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( id__in=valid_urls).order_by('mirror__id', 'url') - delays = url_delays(cutoff_time) + delays = url_delays(cutoff_time, mirror_id) if urls: url_data = dict((item['id'], item) for item in url_data) @@ -90,8 +90,8 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): last_check = max([u.last_check for u in urls]) num_checks = max([u.check_count for u in urls]) check_info = MirrorLog.objects.filter(check_time__gte=cutoff_time) - if mirror_ids: - check_info = check_info.filter(url__mirror_id__in=mirror_ids) + if mirror_id: + check_info = check_info.filter(url__mirror_id=mirror_id) check_info = check_info.aggregate( mn=Min('check_time'), mx=Max('check_time')) if num_checks > 1: @@ -120,7 +120,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): @cache_function(117) -def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None): +def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None): cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( is_success=False, check_time__gte=cutoff_time, @@ -130,8 +130,8 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None): error_count=Count('error'), last_occurred=Max('check_time') ).order_by('-last_occurred', '-error_count') - if mirror_ids: - urls = urls.filter(mirror_id__in=mirror_ids) + if mirror_id: + urls = urls.filter(mirror_id=mirror_id) errors = list(errors) for err in errors: diff --git a/mirrors/views.py b/mirrors/views.py index 56397633..07e28d40 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -175,7 +175,7 @@ def mirror_details(request, name): (not mirror.public or not mirror.active): raise Http404 - status_info = get_mirror_statuses(mirror_ids=[mirror.id]) + status_info = get_mirror_statuses(mirror_id=mirror.id) checked_urls = {url for url in status_info['urls'] \ if url.mirror_id == mirror.id} all_urls = set(mirror.urls.select_related('protocol')) @@ -193,7 +193,7 @@ def mirror_details(request, name): def mirror_details_json(request, name): mirror = get_object_or_404(Mirror, name=name) - status_info = get_mirror_statuses(mirror_ids=[mirror.id]) + status_info = get_mirror_statuses(mirror_id=mirror.id) data = status_info.copy() data['version'] = 3 to_json = json.dumps(data, ensure_ascii=False, -- cgit v1.2.3-2-g168b From 213aa3a2fab6f3a56be348a067c132f568efbaff Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 14 Apr 2013 13:09:41 -0500 Subject: Reduce mirror status query madness Move completely to custom SQL for this logic. The Django ORM just doesn't play nice with the kind of query we are looking to do, so it is easier to do using raw SQL. The biggest pain factor here is in supporting sqlite as it doesn't have nearly the capabilities in handling datetime types directly in the database, as well as having some different type conversion necessities. Signed-off-by: Dan McGee --- mirrors/utils.py | 146 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 91 insertions(+), 55 deletions(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index d18dc22f..eb1211f1 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -2,6 +2,7 @@ from datetime import timedelta from django.db import connection from django.db.models import Avg, Count, Max, Min, StdDev +from django.utils.dateparse import parse_datetime from django.utils.timezone import now from django_countries.fields import Country @@ -11,47 +12,103 @@ from .models import MirrorLog, MirrorProtocol, MirrorUrl DEFAULT_CUTOFF = timedelta(hours=24) -def annotate_url(url, delay): - '''Given a MirrorURL object, add a few more attributes to it regarding - status, including completion_pct, delay, and score.''' - url.completion_pct = float(url.success_count) / url.check_count - if delay is not None: - url.delay = delay - hours = url.delay.days * 24.0 + url.delay.seconds / 3600.0 - if url.completion_pct > 0: - divisor = url.completion_pct - else: - # arbitrary small value - divisor = 0.005 - url.score = (hours + url.duration_avg + url.duration_stddev) / divisor - else: - url.delay = None - url.score = None +def dictfetchall(cursor): + "Returns all rows from a cursor as a dict." + desc = cursor.description + return [ + dict(zip([col[0] for col in desc], row)) + for row in cursor.fetchall() + ] -def url_delays(cutoff_time, mirror_id=None): - cursor = connection.cursor() - if mirror_id is None: - sql= """ -SELECT url_id, AVG(check_time - last_sync) -FROM mirrors_mirrorlog -WHERE is_success = %s AND check_time >= %s AND last_sync IS NOT NULL -GROUP BY url_id +def status_data(cutoff_time, mirror_id=None): + if mirror_id is not None: + params = [cutoff_time, mirror_id] + mirror_where = 'AND u.mirror_id = %s' + else: + params = [cutoff_time] + mirror_where = '' + + vendor = database_vendor(MirrorUrl) + if vendor == 'sqlite': + sql = """ +SELECT l.url_id, u.mirror_id, + COUNT(l.id) AS check_count, + COUNT(l.duration) AS success_count, + MAX(l.last_sync) AS last_sync, + MAX(l.check_time) AS last_check, + AVG(l.duration) AS duration_avg, + 0.0 AS duration_stddev, + AVG(STRFTIME('%%s', check_time) - STRFTIME('%%s', last_sync)) AS delay +FROM mirrors_mirrorlog l +JOIN mirrors_mirrorurl u ON u.id = l.url_id +WHERE l.check_time >= %s +""" + mirror_where + """ +GROUP BY l.url_id, u.mirror_id """ - cursor.execute(sql, [True, cutoff_time]) else: sql = """ -SELECT l.url_id, avg(check_time - last_sync) +SELECT l.url_id, u.mirror_id, + COUNT(l.id) AS check_count, + COUNT(l.duration) AS success_count, + MAX(l.last_sync) AS last_sync, + MAX(l.check_time) AS last_check, + AVG(l.duration) AS duration_avg, + STDDEV(l.duration) AS duration_stddev, + AVG(check_time - last_sync) AS delay FROM mirrors_mirrorlog l JOIN mirrors_mirrorurl u ON u.id = l.url_id -WHERE is_success = %s AND check_time >= %s AND last_sync IS NOT NULL -AND mirror_id = %s -GROUP BY url_id +WHERE l.check_time >= %s +""" + mirror_where + """ +GROUP BY l.url_id, u.mirror_id """ - cursor.execute(sql, [True, cutoff_time, mirror_id]) - return {url_id: delay for url_id, delay in cursor.fetchall()} + cursor = connection.cursor() + cursor.execute(sql, params) + url_data = dictfetchall(cursor) + + # sqlite loves to return less than ideal types + if vendor == 'sqlite': + for item in url_data: + item['delay'] = timedelta(seconds=item['delay']) + item['last_sync'] = parse_datetime(item['last_sync']) + item['last_check'] = parse_datetime(item['last_check']) + + return {item['url_id']: item for item in url_data} + + +def annotate_url(url, url_data): + '''Given a MirrorURL object, add a few more attributes to it regarding + status, including completion_pct, delay, and score.''' + known_attrs = ( + ('success_count', 0), + ('check_count', 0), + ('completion_pct', None), + ('last_check', None), + ('last_sync', None), + ('delay', None), + ('score', None), + ) + for k, v in known_attrs: + setattr(url, k, v) + for k, v in url_data.items(): + if k not in ('url_id', 'mirror_id'): + setattr(url, k, v) + + if url.check_count > 0: + url.completion_pct = float(url.success_count) / url.check_count + + if url.delay is not None: + hours = url.delay.days * 24.0 + url.delay.seconds / 3600.0 + + if url.completion_pct > 0: + divisor = url.completion_pct + else: + # arbitrary small value + divisor = 0.005 + stddev = url.duration_stddev or 0.0 + url.score = (hours + url.duration_avg + stddev) / divisor @cache_function(123) @@ -65,29 +122,14 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): if mirror_id: valid_urls = valid_urls.filter(mirror_id=mirror_id) - url_data = MirrorUrl.objects.values('id', 'mirror_id').filter( - id__in=valid_urls, logs__check_time__gte=cutoff_time).annotate( - check_count=Count('logs'), - success_count=Count('logs__duration'), - last_sync=Max('logs__last_sync'), - last_check=Max('logs__check_time'), - duration_avg=Avg('logs__duration')) - - vendor = database_vendor(MirrorUrl) - if vendor != 'sqlite': - url_data = url_data.annotate(duration_stddev=StdDev('logs__duration')) - + url_data = status_data(cutoff_time, mirror_id) urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( id__in=valid_urls).order_by('mirror__id', 'url') - delays = url_delays(cutoff_time, mirror_id) if urls: - url_data = dict((item['id'], item) for item in url_data) for url in urls: - for k, v in url_data.get(url.id, {}).items(): - if k not in ('id', 'mirror_id'): - setattr(url, k, v) - last_check = max([u.last_check for u in urls]) + annotate_url(url, url_data.get(url.id, {})) + last_check = max([u.last_check for u in urls if u.last_check]) num_checks = max([u.check_count for u in urls]) check_info = MirrorLog.objects.filter(check_time__gte=cutoff_time) if mirror_id: @@ -104,12 +146,6 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): num_checks = 0 check_frequency = None - for url in urls: - # fake the standard deviation for local testing setups - if vendor == 'sqlite': - setattr(url, 'duration_stddev', 0.0) - annotate_url(url, delays.get(url.id, None)) - return { 'cutoff': cutoff, 'last_check': last_check, -- cgit v1.2.3-2-g168b From f357a39a49a8edc713d512976a0be2a2a8ac5c4f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 14 Apr 2013 13:21:16 -0500 Subject: Remove cache_function decorator from a few spots The benefit of these storage operations might be outweighed by the cost, especially given how infrequently these functions are called. Signed-off-by: Dan McGee --- mirrors/utils.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index eb1211f1..5a8bbf5d 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -111,7 +111,6 @@ def annotate_url(url, url_data): url.score = (hours + url.duration_avg + stddev) / divisor -@cache_function(123) def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): cutoff_time = now() - cutoff @@ -155,7 +154,6 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None): } -@cache_function(117) def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None): cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( -- cgit v1.2.3-2-g168b From 0cad22a5eca20ecb64b04d0912592ea6a5361e0d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 14 Apr 2013 13:59:49 -0500 Subject: Add a JSON view for retrieving mirror check locations Signed-off-by: Dan McGee --- mirrors/urls.py | 1 + mirrors/views.py | 33 +++++++++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) (limited to 'mirrors') diff --git a/mirrors/urls.py b/mirrors/urls.py index 4e929410..7cf76aa1 100644 --- a/mirrors/urls.py +++ b/mirrors/urls.py @@ -6,6 +6,7 @@ urlpatterns = patterns('mirrors.views', (r'^status/json/$', 'status_json', {}, 'mirror-status-json'), (r'^status/tier/(?P\d+)/$', 'status', {}, 'mirror-status-tier'), (r'^status/tier/(?P\d+)/json/$', 'status_json', {}, 'mirror-status-tier-json'), + (r'^locations/json/$', 'locations_json', {}, 'mirror-locations-json'), (r'^(?P[\.\-\w]+)/$', 'mirror_details'), (r'^(?P[\.\-\w]+)/json/$', 'mirror_details_json'), ) diff --git a/mirrors/views.py b/mirrors/views.py index 07e28d40..30f96b63 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -13,7 +13,8 @@ from django.utils.timezone import now from django.views.decorators.csrf import csrf_exempt from django_countries.countries import COUNTRIES -from .models import Mirror, MirrorUrl, MirrorProtocol, MirrorLog +from .models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog, + CheckLocation) from .utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF COUNTRY_LOOKUP = dict(COUNTRIES) @@ -264,7 +265,8 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder): class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): '''Adds URL check history information.''' - log_attributes = ('check_time', 'last_sync', 'duration', 'is_success') + log_attributes = ('check_time', 'last_sync', 'duration', 'is_success', + 'location_id') def default(self, obj): if isinstance(obj, MirrorUrl): @@ -292,4 +294,31 @@ def status_json(request, tier=None): response = HttpResponse(to_json, content_type='application/json') return response + +class LocationJSONEncoder(DjangoJSONEncoder): + '''Base JSONEncoder extended to handle CheckLocation objects.''' + + def default(self, obj): + if hasattr(obj, '__iter__'): + # mainly for queryset serialization + return list(obj) + if isinstance(obj, CheckLocation): + return { + 'hostname': obj.hostname, + 'source_ip': obj.source_ip, + 'country': unicode(obj.country.name), + 'country_code': obj.country.code, + 'ip_version': obj.ip_version, + } + return super(LocationJSONEncoder, self).default(obj) + + +def locations_json(request): + data = {} + data['version'] = 1 + data['locations'] = CheckLocation.objects.all() + to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From 9b07cb1ebdc8c5cc5dff66a7edb02e0ddc9f4733 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 14 Apr 2013 15:08:25 -0500 Subject: Draw one mirror status graph per check location Rather than lump it all together and have odd spikes depending on which side of the Atlantic checked a mirror in a given timeslot, draw a chart per check location. Signed-off-by: Dan McGee --- mirrors/static/mirror_status.js | 68 +++++++++++++++++++++++++++++------------ mirrors/views.py | 3 +- 2 files changed, 51 insertions(+), 20 deletions(-) (limited to 'mirrors') diff --git a/mirrors/static/mirror_status.js b/mirrors/static/mirror_status.js index 8ec85c40..4a57128a 100644 --- a/mirrors/static/mirror_status.js +++ b/mirrors/static/mirror_status.js @@ -1,7 +1,16 @@ -function mirror_status(chart_id, data_url) { - var jq_div = jQuery(chart_id); +function draw_graphs(location_url, log_url, container_id) { + $.when($.getJSON(location_url), $.getJSON(log_url)) + .then(function(loc_data, log_data) { + $.each(loc_data[0].locations, function(i, val) { + mirror_status(container_id, val, log_data[0]); + }); + }); +} - var draw_graph = function(data) { +function mirror_status(container_id, check_loc, log_data) { + + var draw_graph = function(chart_id, data) { + var jq_div = jQuery(chart_id); var margin = {top: 20, right: 20, bottom: 30, left: 40}, width = jq_div.width() - margin.left - margin.right, height = jq_div.height() - margin.top - margin.bottom; @@ -106,31 +115,52 @@ function mirror_status(chart_id, data_url) { .text(function(d) { return d; }); }; - /* invoke the data-fetch + first draw */ - var cached_data = null; - d3.json(data_url, function(json) { - cached_data = jQuery.map(json.urls, function(url, i) { + var filter_data = function(json, location_id) { + return jQuery.map(json.urls, function(url, i) { + var logs = jQuery.map(url.logs, function(log, j) { + if (!log.is_success) { + return null; + } + /* screen by location ID if we were given one */ + if (location_id && log.location_id !== location_id) { + return null; + } + return { + duration: log.duration, + check_time: new Date(log.check_time) + }; + }); + /* don't return URLs without any log info */ + if (logs.length === 0) { + return null; + } return { url: url.url, - logs: jQuery.map(url.logs, function(log, j) { - if (!log.is_success) { - return null; - } - return { - duration: log.duration, - check_time: new Date(log.check_time) - }; - }) + logs: logs }; }); - draw_graph(cached_data); - }); + }; + + var cached_data = filter_data(log_data, check_loc.id); + /* we had a check location with no log data handed to us, skip graphing */ + if (cached_data.length === 0) { + return; + } + + /* create the containers, defer the actual graph drawing */ + var chart_id = 'status-chart-' + check_loc.id; + $(container_id).append('

' + check_loc.country + ' (' + check_loc.source_ip + '), IPv' + check_loc.ip_version + '

'); + $(container_id).append('
'); + $(container_id).append('
'); + setTimeout(function() { + draw_graph('#' + chart_id, cached_data); + }, 0); /* then hook up a resize handler to redraw if necessary */ var resize_timeout = null; var real_resize = function() { resize_timeout = null; - draw_graph(cached_data); + draw_graph('#' + chart_id, cached_data); }; jQuery(window).resize(function() { if (resize_timeout) { diff --git a/mirrors/views.py b/mirrors/views.py index 30f96b63..9311fb8f 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -304,6 +304,7 @@ class LocationJSONEncoder(DjangoJSONEncoder): return list(obj) if isinstance(obj, CheckLocation): return { + 'id': obj.pk, 'hostname': obj.hostname, 'source_ip': obj.source_ip, 'country': unicode(obj.country.name), @@ -316,7 +317,7 @@ class LocationJSONEncoder(DjangoJSONEncoder): def locations_json(request): data = {} data['version'] = 1 - data['locations'] = CheckLocation.objects.all() + data['locations'] = CheckLocation.objects.all().order_by('pk') to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder) response = HttpResponse(to_json, content_type='application/json') return response -- cgit v1.2.3-2-g168b From 4fd50fa622b13ecc5104919c0e7ed51f64734d92 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 14 Apr 2013 15:23:09 -0500 Subject: Tweaks to mirror status chart generation * Use 'jQuery' rather than '$' * Use same colors for URLs in every chart for clarity Signed-off-by: Dan McGee --- mirrors/static/mirror_status.js | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) (limited to 'mirrors') diff --git a/mirrors/static/mirror_status.js b/mirrors/static/mirror_status.js index 4a57128a..241f5c61 100644 --- a/mirrors/static/mirror_status.js +++ b/mirrors/static/mirror_status.js @@ -1,13 +1,15 @@ function draw_graphs(location_url, log_url, container_id) { - $.when($.getJSON(location_url), $.getJSON(log_url)) + jQuery.when(jQuery.getJSON(location_url), jQuery.getJSON(log_url)) .then(function(loc_data, log_data) { - $.each(loc_data[0].locations, function(i, val) { - mirror_status(container_id, val, log_data[0]); + /* use the same color selection for a given URL in every graph */ + var color = d3.scale.category10(); + jQuery.each(loc_data[0].locations, function(i, val) { + mirror_status(container_id, val, log_data[0], color); }); }); } -function mirror_status(container_id, check_loc, log_data) { +function mirror_status(container_id, check_loc, log_data, color) { var draw_graph = function(chart_id, data) { var jq_div = jQuery(chart_id); @@ -15,8 +17,7 @@ function mirror_status(container_id, check_loc, log_data) { width = jq_div.width() - margin.left - margin.right, height = jq_div.height() - margin.top - margin.bottom; - var color = d3.scale.category10(), - x = d3.time.scale.utc().range([0, width]), + var x = d3.time.scale.utc().range([0, width]), y = d3.scale.linear().range([height, 0]), x_axis = d3.svg.axis().scale(x).orient("bottom"), y_axis = d3.svg.axis().scale(y).orient("left"); @@ -95,8 +96,9 @@ function mirror_status(container_id, check_loc, log_data) { .text(function(d) { return d.url + "\n" + d.duration.toFixed(3) + " secs\n" + d.check_time.toUTCString(); }); /* add a legend for good measure */ + var active = jQuery.map(data, function(item, i) { return item.url; }); var legend = svg.selectAll(".legend") - .data(color.domain()) + .data(active) .enter().append("g") .attr("class", "legend") .attr("transform", function(d, i) { return "translate(0," + i * 20 + ")"; }); @@ -149,9 +151,9 @@ function mirror_status(container_id, check_loc, log_data) { /* create the containers, defer the actual graph drawing */ var chart_id = 'status-chart-' + check_loc.id; - $(container_id).append('

' + check_loc.country + ' (' + check_loc.source_ip + '), IPv' + check_loc.ip_version + '

'); - $(container_id).append('
'); - $(container_id).append('
'); + jQuery(container_id).append('

' + check_loc.country + ' (' + check_loc.source_ip + '), IPv' + check_loc.ip_version + '

'); + jQuery(container_id).append('
'); + jQuery(container_id).append('
'); setTimeout(function() { draw_graph('#' + chart_id, cached_data); }, 0); -- cgit v1.2.3-2-g168b From 0589853360d12a1746e2d8e92e798f2727a0b5df Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 16 Apr 2013 21:52:20 -0500 Subject: Remove COUNTRY_LOOKUP global variable This is only used in one place, so it makes more sense for it to not be globally accessible. Signed-off-by: Dan McGee --- mirrors/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'mirrors') diff --git a/mirrors/views.py b/mirrors/views.py index 9311fb8f..73d40297 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -17,8 +17,6 @@ from .models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog, CheckLocation) from .utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF -COUNTRY_LOOKUP = dict(COUNTRIES) - class MirrorlistForm(forms.Form): country = forms.MultipleChoiceField(required=False) @@ -29,6 +27,8 @@ class MirrorlistForm(forms.Form): widget=CheckboxSelectMultiple) use_mirror_status = forms.BooleanField(required=False) + countries = dict(COUNTRIES) + def __init__(self, *args, **kwargs): super(MirrorlistForm, self).__init__(*args, **kwargs) fields = self.fields @@ -46,7 +46,7 @@ class MirrorlistForm(forms.Form): country_codes.update(MirrorUrl.objects.filter( mirror__active=True).exclude(country='').values_list( 'country', flat=True).order_by().distinct()) - countries = [(code, COUNTRY_LOOKUP[code]) for code in country_codes] + countries = [(code, self.countries[code]) for code in country_codes] return sorted(countries, key=itemgetter(1)) def as_div(self): -- cgit v1.2.3-2-g168b From b7b24740640e24883cd17fd683e1d465fbb343f8 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 16 Apr 2013 22:12:01 -0500 Subject: Various minor code cleanups and fixes Most of these were suggested by PyCharm, and include everything from little syntax issues and other bad smells to dead or bad code. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 12 +++--------- mirrors/models.py | 2 +- mirrors/utils.py | 6 +++--- 3 files changed, 7 insertions(+), 13 deletions(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index d6de8f22..e7dd7b49 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -106,19 +106,13 @@ def parse_lastsync(log, data): def check_mirror_url(mirror_url, location, timeout): - if location: - if location.family == socket.AF_INET6: - ipopt = '--ipv6' - elif location.family == socket.AF_INET: - ipopt = '--ipv4' - url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) log = MirrorLog(url=mirror_url, check_time=now(), location=location) headers = {'User-Agent': 'archweb/1.0'} req = urllib2.Request(url, None, headers) + start = time.time() try: - start = time.time() result = urllib2.urlopen(req, timeout=timeout) data = result.read() result.close() @@ -147,12 +141,12 @@ def check_mirror_url(mirror_url, location, timeout): elif isinstance(e.reason, socket.error): log.error = e.reason.args[1] logger.debug("failed: %s, %s", url, log.error) - except HTTPException as e: + except HTTPException: # e.g., BadStatusLine log.is_success = False log.error = "Exception in processing HTTP request." logger.debug("failed: %s, %s", url, log.error) - except socket.timeout as e: + except socket.timeout: log.is_success = False log.error = "Connection timed out." logger.debug("failed: %s, %s", url, log.error) diff --git a/mirrors/models.py b/mirrors/models.py index 791b0078..d8ac7952 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -92,7 +92,7 @@ class MirrorUrl(models.Model): families = self.address_families() self.has_ipv4 = socket.AF_INET in families self.has_ipv6 = socket.AF_INET6 in families - except socket.error as e: + except socket.error: # We don't fail in this case; we'll just set both to False self.has_ipv4 = False self.has_ipv6 = False diff --git a/mirrors/utils.py b/mirrors/utils.py index 5a8bbf5d..531cf005 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -1,13 +1,13 @@ from datetime import timedelta from django.db import connection -from django.db.models import Avg, Count, Max, Min, StdDev +from django.db.models import Count, Max, Min from django.utils.dateparse import parse_datetime from django.utils.timezone import now from django_countries.fields import Country from main.utils import cache_function, database_vendor -from .models import MirrorLog, MirrorProtocol, MirrorUrl +from .models import MirrorLog, MirrorUrl DEFAULT_CUTOFF = timedelta(hours=24) @@ -165,7 +165,7 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None): ).order_by('-last_occurred', '-error_count') if mirror_id: - urls = urls.filter(mirror_id=mirror_id) + errors = errors.filter(url__mirror_id=mirror_id) errors = list(errors) for err in errors: -- cgit v1.2.3-2-g168b From 6de0cfbd23aae69036439db817cc26740d8796cd Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 20 Apr 2013 11:13:49 -0500 Subject: Fix some None issues with sqlite3 and mirror status If certain attributes came back from the database as NULL, we had issues parsing them. Pass None/NULL straight through rather than trying to type-convert. Signed-off-by: Dan McGee --- mirrors/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index 531cf005..ba45da5f 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -71,8 +71,10 @@ GROUP BY l.url_id, u.mirror_id # sqlite loves to return less than ideal types if vendor == 'sqlite': for item in url_data: - item['delay'] = timedelta(seconds=item['delay']) - item['last_sync'] = parse_datetime(item['last_sync']) + if item['delay'] is not None: + item['delay'] = timedelta(seconds=item['delay']) + if item['last_sync'] is not None: + item['last_sync'] = parse_datetime(item['last_sync']) item['last_check'] = parse_datetime(item['last_check']) return {item['url_id']: item for item in url_data} -- cgit v1.2.3-2-g168b