From 8d21e9f4b1d5a31880f9973d758de9becc90eb39 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 28 Apr 2012 18:25:37 -0500 Subject: mirrorresolv: only run update query if values changed 98% of the time, we won't need to update the existing values as it will be the same as the prior run of this command. Do a quick check of the old and new values and don't send anything to the database if there is no need for an update. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorresolv.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorresolv.py b/mirrors/management/commands/mirrorresolv.py index 4e812f2d..0370f8ed 100644 --- a/mirrors/management/commands/mirrorresolv.py +++ b/mirrors/management/commands/mirrorresolv.py @@ -41,13 +41,19 @@ def resolve_mirrors(): logger.debug("requesting list of mirror URLs") for mirrorurl in MirrorUrl.objects.filter(mirror__active=True): try: + # save old values, we can skip no-op updates this way + oldvals = (mirrorurl.has_ipv4, mirrorurl.has_ipv6) logger.debug("resolving %3i (%s)", mirrorurl.id, mirrorurl.hostname) families = mirrorurl.address_families() mirrorurl.has_ipv4 = socket.AF_INET in families mirrorurl.has_ipv6 = socket.AF_INET6 in families logger.debug("%s: v4: %s v6: %s", mirrorurl.hostname, mirrorurl.has_ipv4, mirrorurl.has_ipv6) - mirrorurl.save(force_update=True) + # now check new values, only update if new != old + newvals = (mirrorurl.has_ipv4, mirrorurl.has_ipv6) + if newvals != oldvals: + logger.debug("values changed for %s", mirrorurl) + mirrorurl.save(force_update=True) except socket.error, e: logger.warn("error resolving %s: %s", mirrorurl.hostname, e) -- cgit v1.2.3-2-g168b From 44eb2d5ee0fa9e1b495027cec3e663ff85c0ed1d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 29 Apr 2012 21:26:23 -0500 Subject: Use a custom User-Agent when checking mirror URLs Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 7ffb7773..c1269226 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -35,6 +35,7 @@ logging.basicConfig( stream=sys.stderr) logger = logging.getLogger() + class Command(NoArgsCommand): help = "Runs a check on all known mirror URLs to determine their up-to-date status." @@ -49,13 +50,16 @@ class Command(NoArgsCommand): return check_current_mirrors() + def check_mirror_url(mirror_url): url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) log = MirrorLog(url=mirror_url, check_time=utc_now()) + headers = {'User-Agent': 'archweb/1.0'} + req = urllib2.Request(url, None, headers) try: start = time.time() - result = urllib2.urlopen(url, timeout=10) + result = urllib2.urlopen(req, timeout=10) data = result.read() result.close() end = time.time() @@ -104,6 +108,7 @@ def check_mirror_url(mirror_url): return log + def mirror_url_worker(work, output): while True: try: @@ -116,11 +121,12 @@ def mirror_url_worker(work, output): except Empty: return 0 + class MirrorCheckPool(object): - def __init__(self, work, num_threads=10): + def __init__(self, urls, num_threads=10): self.tasks = Queue() self.logs = deque() - for i in list(work): + for i in list(urls): self.tasks.put(i) self.threads = [] for i in range(num_threads): @@ -140,6 +146,7 @@ class MirrorCheckPool(object): MirrorLog.objects.bulk_create(self.logs) logger.debug("log entries saved") + def check_current_mirrors(): urls = MirrorUrl.objects.filter( protocol__is_download=True, @@ -149,8 +156,4 @@ def check_current_mirrors(): pool.run() return 0 -# For lack of a better place to put it, here is a query to get latest check -# result joined with mirror details: -# SELECT mu.*, m.*, ml.* FROM mirrors_mirrorurl mu JOIN mirrors_mirror m ON mu.mirror_id = m.id JOIN mirrors_mirrorlog ml ON mu.id = ml.url_id LEFT JOIN mirrors_mirrorlog ml2 ON ml.url_id = ml2.url_id AND ml.id < ml2.id WHERE ml2.id IS NULL AND m.active = 1 AND m.public = 1; - # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From d2d0895f13835569ff25a3161ddb94cd655dfd4f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 2 May 2012 12:23:21 -0500 Subject: Allow mirrorlist generator pattern to match any protocol Add a helper method that checks if we know about the protocol; if so, we can spit out a URL for it. This allows (if you are insane) generation of an rsync mirrorlist, for instance. Signed-off-by: Dan McGee --- mirrors/urls_mirrorlist.py | 5 +---- mirrors/views.py | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 9 deletions(-) (limited to 'mirrors') diff --git a/mirrors/urls_mirrorlist.py b/mirrors/urls_mirrorlist.py index e0f44c78..1444eca9 100644 --- a/mirrors/urls_mirrorlist.py +++ b/mirrors/urls_mirrorlist.py @@ -3,10 +3,7 @@ from django.conf.urls import patterns urlpatterns = patterns('mirrors.views', (r'^$', 'generate_mirrorlist', {}, 'mirrorlist'), (r'^all/$', 'find_mirrors', {'countries': ['all']}), - (r'^all/ftp/$', 'find_mirrors', - {'countries': ['all'], 'protocols': ['ftp']}), - (r'^all/http/$', 'find_mirrors', - {'countries': ['all'], 'protocols': ['http']}), + (r'^all/(?P[A-z]+)/$', 'find_mirrors_simple') ) # vim: set ts=4 sw=4 et: diff --git a/mirrors/views.py b/mirrors/views.py index c52656f7..6f37ace1 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -82,12 +82,15 @@ def generate_mirrorlist(request): def find_mirrors(request, countries=None, protocols=None, use_status=False, ipv4_supported=True, ipv6_supported=True): if not protocols: - protocols = MirrorProtocol.objects.filter( - is_download=True).values_list('protocol', flat=True) + protocols = MirrorProtocol.objects.filter(is_download=True) + elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol: + # we already have a queryset, no need to query again + pass + else: + protocols = MirrorProtocol.objects.filter(protocol__in=protocols) qset = MirrorUrl.objects.select_related().filter( - protocol__protocol__in=protocols, - mirror__public=True, mirror__active=True, - ) + protocol__in=protocols, + mirror__public=True, mirror__active=True) if countries and 'all' not in countries: qset = qset.filter(Q(country__in=countries) | Q(mirror__country__in=countries)) @@ -124,6 +127,11 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, mimetype='text/plain') +def find_mirrors_simple(request, protocol): + proto = get_object_or_404(MirrorProtocol, protocol=protocol) + return find_mirrors(request, protocols=[proto]) + + def mirrors(request): mirror_list = Mirror.objects.select_related().order_by('tier', 'country') if not request.user.is_authenticated(): -- cgit v1.2.3-2-g168b From 12bf4c1b1e7df2d934b9dfde8629137dedeea99f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 18 Feb 2012 19:17:00 -0600 Subject: Add a smart protocol filter to mirrorlist generator This will only list FTP mirrors for a given country if there are no HTTP mirrors available, since FTP must die. Signed-off-by: Dan McGee --- mirrors/views.py | 55 ++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 15 deletions(-) (limited to 'mirrors') diff --git a/mirrors/views.py b/mirrors/views.py index 6f37ace1..eac78ff2 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -1,4 +1,5 @@ from datetime import timedelta +from itertools import groupby from operator import attrgetter, itemgetter from django import forms @@ -79,8 +80,34 @@ def generate_mirrorlist(request): {'mirrorlist_form': form}) +def default_protocol_filter(original_urls): + key_func = attrgetter('real_country') + sorted_urls = sorted(original_urls, key=key_func) + urls = [] + for _, group in groupby(sorted_urls, key=key_func): + cntry_urls = list(group) + if any(url.protocol.default for url in cntry_urls): + cntry_urls = [url for url in cntry_urls if url.protocol.default] + urls.extend(cntry_urls) + return urls + + +def status_filter(original_urls): + status_info = get_mirror_statuses() + scores = dict((u.id, u.score) for u in status_info['urls']) + urls = [] + for u in original_urls: + u.score = scores.get(u.id, None) + # also include mirrors that don't have an up to date score + # (as opposed to those that have been set with no score) + if (u.id not in scores) or (u.score and u.score < 100.0): + urls.append(u) + # if a url doesn't have a score, treat it as the highest possible + return sorted(urls, key=lambda x: x.score or 100.0) + + def find_mirrors(request, countries=None, protocols=None, use_status=False, - ipv4_supported=True, ipv6_supported=True): + ipv4_supported=True, ipv6_supported=True, smart_protocol=False): if not protocols: protocols = MirrorProtocol.objects.filter(is_download=True) elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol: @@ -102,23 +129,17 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, ip_version |= Q(has_ipv6=True) qset = qset.filter(ip_version) + if smart_protocol: + urls = default_protocol_filter(qset) + else: + urls = qset + if not use_status: - urls = qset.order_by('mirror__name', 'url') - urls = sorted(urls, key=attrgetter('real_country')) + sort_key = attrgetter('real_country', 'mirror.name', 'url') + urls = sorted(urls, key=sort_key) template = 'mirrors/mirrorlist.txt' else: - status_info = get_mirror_statuses() - scores = dict([(u.id, u.score) for u in status_info['urls']]) - urls = [] - for u in qset: - u.score = scores.get(u.id, None) - # also include mirrors that don't have an up to date score - # (as opposed to those that have been set with no score) - if (u.id not in scores) or \ - (u.score and u.score < 100.0): - urls.append(u) - # if a url doesn't have a score, treat it as the highest possible - urls = sorted(urls, key=lambda x: x.score or 100.0) + urls = status_filter(urls) template = 'mirrors/mirrorlist_status.txt' return direct_to_template(request, template, { @@ -128,6 +149,10 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, def find_mirrors_simple(request, protocol): + if protocol == 'smart': + # generate a 'smart' mirrorlist, one that only includes FTP mirrors if + # no HTTP mirror is available in that country. + return find_mirrors(request, smart_protocol=True) proto = get_object_or_404(MirrorProtocol, protocol=protocol) return find_mirrors(request, protocols=[proto]) -- cgit v1.2.3-2-g168b From a5f5557493446bede78adb0584c88208234f874e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 12 May 2012 09:32:30 -0500 Subject: Use python json module directly in place of simplejson As of Python 2.6, this is a builtin module that has all the same functions and capabilities of the Django simplejson module. Additionally simplejson is deprecated in the upcoming Django 1.5 release. Signed-off-by: Dan McGee --- mirrors/views.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'mirrors') diff --git a/mirrors/views.py b/mirrors/views.py index eac78ff2..b0be6238 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -1,5 +1,6 @@ from datetime import timedelta from itertools import groupby +import json from operator import attrgetter, itemgetter from django import forms @@ -10,7 +11,6 @@ from django.http import Http404, HttpResponse from django.shortcuts import get_object_or_404 from django.views.decorators.csrf import csrf_exempt from django.views.generic.simple import direct_to_template -from django.utils import simplejson from django_countries.countries import COUNTRIES from .models import Mirror, MirrorUrl, MirrorProtocol @@ -237,8 +237,7 @@ def status_json(request): status_info = get_mirror_statuses() data = status_info.copy() data['version'] = 3 - to_json = simplejson.dumps(data, ensure_ascii=False, - cls=MirrorStatusJSONEncoder) + to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder) response = HttpResponse(to_json, mimetype='application/json') return response -- cgit v1.2.3-2-g168b From 76516cae45f3d1080065608bdb8f2d086322012f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 13 May 2012 19:45:57 -0500 Subject: Don't limit protocols returned by mirror status function If results weren't available for certain URLs, they won't show up anyway in this list, and if we start to check rsync URLs, then we want their values to come back in this status list. Signed-off-by: Dan McGee --- mirrors/utils.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index 32fa3587..54de567e 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -32,11 +32,9 @@ def annotate_url(url, delays): @cache_function(123) def get_mirror_statuses(cutoff=default_cutoff): cutoff_time = utc_now() - cutoff - protocols = list(MirrorProtocol.objects.filter(is_download=True)) # I swear, this actually has decent performance... urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( mirror__active=True, mirror__public=True, - protocol__in=protocols, logs__check_time__gte=cutoff_time).annotate( check_count=Count('logs'), success_count=Count('logs__duration'), -- cgit v1.2.3-2-g168b From a570c2c7fa404c2eb1760c6fb428047083c0c957 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 13 May 2012 20:10:46 -0500 Subject: Change mirror log error text to unlimited length Use TextField rather than a limited-length CharField; leave it up to the code filling this field to determine an appropriate length. Signed-off-by: Dan McGee --- .../0017_auto__chg_field_mirrorlog_error.py | 66 ++++++++++++++++++++++ mirrors/models.py | 6 +- 2 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py (limited to 'mirrors') diff --git a/mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py b/mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py new file mode 100644 index 00000000..2f76c099 --- /dev/null +++ b/mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.alter_column('mirrors_mirrorlog', 'error', self.gf('django.db.models.fields.TextField')()) + + def backwards(self, orm): + db.alter_column('mirrors_mirrorlog', 'error', self.gf('django.db.models.fields.CharField')(max_length=255)) + + models = { + 'mirrors.mirror': { + 'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + 'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) + }, + 'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) + }, + 'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index 19437610..8c2bd7fc 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -44,6 +44,7 @@ class Mirror(models.Model): def get_absolute_url(self): return '/mirrors/%s/' % self.name + class MirrorProtocol(models.Model): protocol = models.CharField(max_length=10, unique=True) is_download = models.BooleanField(default=True, @@ -57,6 +58,7 @@ class MirrorProtocol(models.Model): class Meta: ordering = ('protocol',) + class MirrorUrl(models.Model): url = models.CharField("URL", max_length=255, unique=True) protocol = models.ForeignKey(MirrorProtocol, related_name="urls", @@ -104,6 +106,7 @@ class MirrorUrl(models.Model): class Meta: verbose_name = 'mirror URL' + class MirrorRsync(models.Model): ip = models.CharField("IP", max_length=24) mirror = models.ForeignKey(Mirror, related_name="rsync_ips") @@ -114,13 +117,14 @@ class MirrorRsync(models.Model): class Meta: verbose_name = 'mirror rsync IP' + class MirrorLog(models.Model): url = models.ForeignKey(MirrorUrl, related_name="logs") check_time = models.DateTimeField(db_index=True) last_sync = models.DateTimeField(null=True) duration = models.FloatField(null=True) is_success = models.BooleanField(default=True) - error = models.CharField(max_length=255, blank=True, default='') + error = models.TextField(blank=True, default='') def __unicode__(self): return "Check of %s at %s" % (self.url.url, self.check_time) -- cgit v1.2.3-2-g168b From 2f7d770b261b3428bcff366ba6ff4fa631dd980a Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 13 May 2012 20:15:00 -0500 Subject: Add rsync support to mirrorcheck and other small improvements The main changes in this patch implement rsync:// protocol checking support by calling the rsync binary, requested in FS#29878. We track and log much of the same things as we already do for FTP and HTTP URLs- check time, last sync, total check duration, etc. Also added in this patch is a configurable timeout value which defaults to the previous hardcoded value of 10 seconds; this can be passed as an option to the mirrorcheck command. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 130 ++++++++++++++++++++--------- 1 file changed, 89 insertions(+), 41 deletions(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index c1269226..ae89d5e0 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -9,22 +9,26 @@ we encounter errors, record those as well. Usage: ./manage.py mirrorcheck """ -from django.core.management.base import NoArgsCommand -from django.db import transaction - from collections import deque from datetime import datetime import logging +import os +from optparse import make_option +from pytz import utc import re import socket +import subprocess import sys import time +import tempfile from threading import Thread import types -from pytz import utc from Queue import Queue, Empty import urllib2 +from django.core.management.base import NoArgsCommand +from django.db import transaction + from main.utils import utc_now from mirrors.models import MirrorUrl, MirrorLog @@ -37,10 +41,15 @@ logger = logging.getLogger() class Command(NoArgsCommand): + option_list = NoArgsCommand.option_list + ( + make_option('-t', '--timeout', dest='timeout', default='10', + help='Timeout value for connecting to URL'), + ) help = "Runs a check on all known mirror URLs to determine their up-to-date status." def handle_noargs(self, **options): v = int(options.get('verbosity', 0)) + timeout = int(options.get('timeout', 10)) if v == 0: logger.level = logging.ERROR elif v == 1: @@ -48,10 +57,29 @@ class Command(NoArgsCommand): elif v == 2: logger.level = logging.DEBUG - return check_current_mirrors() + urls = MirrorUrl.objects.select_related('protocol').filter( + mirror__active=True, mirror__public=True) + + pool = MirrorCheckPool(urls, timeout) + pool.run() + return 0 -def check_mirror_url(mirror_url): +def parse_lastsync(log, data): + '''lastsync file should be an epoch value created by us.''' + try: + parsed_time = datetime.utcfromtimestamp(int(data)) + log.last_sync = parsed_time.replace(tzinfo=utc) + except ValueError: + # it is bad news to try logging the lastsync value; + # sometimes we get a crazy-encoded web page. + # if we couldn't parse a time, this is a failure. + log.last_sync = None + log.error = "Could not parse time from lastsync" + log.is_success = False + + +def check_mirror_url(mirror_url, timeout): url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) log = MirrorLog(url=mirror_url, check_time=utc_now()) @@ -59,28 +87,14 @@ def check_mirror_url(mirror_url): req = urllib2.Request(url, None, headers) try: start = time.time() - result = urllib2.urlopen(req, timeout=10) + result = urllib2.urlopen(req, timeout=timeout) data = result.read() result.close() end = time.time() - # lastsync should be an epoch value created by us - parsed_time = None - try: - parsed_time = datetime.utcfromtimestamp(int(data)) - parsed_time = parsed_time.replace(tzinfo=utc) - except ValueError: - # it is bad news to try logging the lastsync value; - # sometimes we get a crazy-encoded web page. - pass - - log.last_sync = parsed_time - # if we couldn't parse a time, this is a failure - if parsed_time is None: - log.error = "Could not parse time from lastsync" - log.is_success = False + parse_lastsync(log, data) log.duration = end - start logger.debug("success: %s, %.2f", url, log.duration) - except urllib2.HTTPError, e: + except urllib2.HTTPError as e: if e.code == 404: # we have a duration, just not a success end = time.time() @@ -88,7 +102,7 @@ def check_mirror_url(mirror_url): log.is_success = False log.error = str(e) logger.debug("failed: %s, %s", url, log.error) - except urllib2.URLError, e: + except urllib2.URLError as e: log.is_success = False log.error = e.reason if isinstance(e.reason, types.StringTypes) and \ @@ -101,20 +115,64 @@ def check_mirror_url(mirror_url): elif isinstance(e.reason, socket.error): log.error = e.reason.args[1] logger.debug("failed: %s, %s", url, log.error) - except socket.timeout, e: + except socket.timeout as e: log.is_success = False log.error = "Connection timed out." logger.debug("failed: %s, %s", url, log.error) + except socket.error as e: + log.is_success = False + log.error = str(e) + logger.debug("failed: %s, %s", url, log.error) + + return log + + +def check_rsync_url(mirror_url, timeout): + url = mirror_url.url + 'lastsync' + logger.info("checking URL %s", url) + log = MirrorLog(url=mirror_url, check_time=utc_now()) + + tempdir = tempfile.mkdtemp() + lastsync_path = os.path.join(tempdir, 'lastsync') + rsync_cmd = ["rsync", "--quiet", "--contimeout=%d" % timeout, + "--timeout=%d" % timeout, url, lastsync_path] + try: + with open(os.devnull, 'w') as devnull: + proc = subprocess.Popen(rsync_cmd, stdout=devnull, + stderr=subprocess.PIPE) + start = time.time() + _, errdata = proc.communicate() + end = time.time() + log.duration = end - start + if proc.returncode != 0: + logger.debug("error: %s, %s", url, errdata) + log.is_success = False + log.error = errdata.strip() + # look at rsync error code- if we had a command error or timed out, + # don't record a duration as it is misleading + if proc.returncode in (1, 30, 35): + log.duration = None + else: + logger.debug("success: %s, %.2f", url, log.duration) + with open(lastsync_path, 'r') as lastsync: + parse_lastsync(log, lastsync.read()) + finally: + if os.path.exists(lastsync_path): + os.unlink(lastsync_path) + os.rmdir(tempdir) return log -def mirror_url_worker(work, output): +def mirror_url_worker(work, output, timeout): while True: try: - item = work.get(block=False) + url = work.get(block=False) try: - log = check_mirror_url(item) + if url.protocol.protocol == 'rsync': + log = check_rsync_url(url, timeout) + else: + log = check_mirror_url(url, timeout) output.append(log) finally: work.task_done() @@ -123,7 +181,7 @@ def mirror_url_worker(work, output): class MirrorCheckPool(object): - def __init__(self, urls, num_threads=10): + def __init__(self, urls, timeout=10, num_threads=10): self.tasks = Queue() self.logs = deque() for i in list(urls): @@ -131,7 +189,7 @@ class MirrorCheckPool(object): self.threads = [] for i in range(num_threads): thread = Thread(target=mirror_url_worker, - args=(self.tasks, self.logs)) + args=(self.tasks, self.logs, timeout)) thread.daemon = True self.threads.append(thread) @@ -142,18 +200,8 @@ class MirrorCheckPool(object): thread.start() logger.debug("joining on all threads") self.tasks.join() - logger.debug("processing log entries") + logger.debug("processing %d log entries", len(self.logs)) MirrorLog.objects.bulk_create(self.logs) logger.debug("log entries saved") - -def check_current_mirrors(): - urls = MirrorUrl.objects.filter( - protocol__is_download=True, - mirror__active=True, mirror__public=True) - - pool = MirrorCheckPool(urls) - pool.run() - return 0 - # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From ae1c526ffbe908322f0dd8d8805360b81ab22b0f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 13 May 2012 20:35:50 -0500 Subject: Add ability to restrict status report to single tier This should make it easier to catch errors in our Tier 1 mirrors. Signed-off-by: Dan McGee --- mirrors/urls.py | 1 + mirrors/utils.py | 2 +- mirrors/views.py | 19 ++++++++++++++++--- 3 files changed, 18 insertions(+), 4 deletions(-) (limited to 'mirrors') diff --git a/mirrors/urls.py b/mirrors/urls.py index f002e9d6..bb4eb969 100644 --- a/mirrors/urls.py +++ b/mirrors/urls.py @@ -4,6 +4,7 @@ urlpatterns = patterns('mirrors.views', (r'^$', 'mirrors', {}, 'mirror-list'), (r'^status/$', 'status', {}, 'mirror-status'), (r'^status/json/$', 'status_json', {}, 'mirror-status-json'), + (r'^status/tier/(?P\d+)/$', 'status', {}, 'mirror-status-tier'), (r'^(?P[\.\-\w]+)/$', 'mirror_details'), ) diff --git a/mirrors/utils.py b/mirrors/utils.py index 54de567e..2014411d 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -88,7 +88,7 @@ def get_mirror_errors(cutoff=default_cutoff): is_success=False, check_time__gte=cutoff_time, url__mirror__active=True, url__mirror__public=True).values( 'url__url', 'url__country', 'url__protocol__protocol', - 'url__mirror__country', 'error').annotate( + 'url__mirror__country', 'url__mirror__tier', 'error').annotate( error_count=Count('error'), last_occurred=Max('check_time') ).order_by('-last_occurred', '-error_count') errors = list(errors) diff --git a/mirrors/views.py b/mirrors/views.py index b0be6238..8f092be7 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -13,7 +13,7 @@ from django.views.decorators.csrf import csrf_exempt from django.views.generic.simple import direct_to_template from django_countries.countries import COUNTRIES -from .models import Mirror, MirrorUrl, MirrorProtocol +from .models import Mirror, MirrorUrl, MirrorProtocol, TIER_CHOICES from .utils import get_mirror_statuses, get_mirror_errors COUNTRY_LOOKUP = dict(COUNTRIES) @@ -184,7 +184,11 @@ def mirror_details(request, name): {'mirror': mirror, 'urls': all_urls}) -def status(request): +def status(request, tier=None): + if tier is not None: + tier = int(tier) + if tier not in [t[0] for t in TIER_CHOICES]: + raise Http404 bad_timedelta = timedelta(days=3) status_info = get_mirror_statuses() @@ -192,17 +196,26 @@ def status(request): good_urls = [] bad_urls = [] for url in urls: + # screen by tier if we were asked to + if tier is not None and url.mirror.tier != tier: + continue # split them into good and bad lists based on delay if not url.delay or url.delay > bad_timedelta: bad_urls.append(url) else: good_urls.append(url) + error_logs = get_mirror_errors() + if tier is not None: + error_logs = [log for log in error_logs + if log['url__mirror__tier'] == tier] + context = status_info.copy() context.update({ 'good_urls': sorted(good_urls, key=attrgetter('score')), 'bad_urls': sorted(bad_urls, key=lambda u: u.delay or timedelta.max), - 'error_logs': get_mirror_errors(), + 'error_logs': error_logs, + 'tier': tier, }) return direct_to_template(request, 'mirrors/status.html', context) -- cgit v1.2.3-2-g168b From 06f1bb99617e532f6b39c135370de79be7c270fa Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 18 May 2012 20:42:05 -0500 Subject: mirrors: add an alternate_email column We have a lot of these in the freeform text area in the mirror notes; attempt to make this data usable as necessary if we want to do some sort of mirror notification automation in the future. Signed-off-by: Dan McGee --- mirrors/admin.py | 4 +- .../0018_auto__add_field_mirror_alternate_email.py | 68 ++++++++++++++++++++++ mirrors/models.py | 1 + 3 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 mirrors/migrations/0018_auto__add_field_mirror_alternate_email.py (limited to 'mirrors') diff --git a/mirrors/admin.py b/mirrors/admin.py index b7b9894c..65fff368 100644 --- a/mirrors/admin.py +++ b/mirrors/admin.py @@ -63,9 +63,9 @@ class MirrorAdminForm(forms.ModelForm): class MirrorAdmin(admin.ModelAdmin): form = MirrorAdminForm list_display = ('name', 'tier', 'country', 'active', 'public', - 'isos', 'admin_email') + 'isos', 'admin_email', 'alternate_email') list_filter = ('tier', 'active', 'public', 'country') - search_fields = ('name',) + search_fields = ('name', 'admin_email', 'alternate_email') inlines = [ MirrorUrlInlineAdmin, MirrorRsyncInlineAdmin, diff --git a/mirrors/migrations/0018_auto__add_field_mirror_alternate_email.py b/mirrors/migrations/0018_auto__add_field_mirror_alternate_email.py new file mode 100644 index 00000000..a08699e8 --- /dev/null +++ b/mirrors/migrations/0018_auto__add_field_mirror_alternate_email.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + def forwards(self, orm): + db.add_column('mirrors_mirror', 'alternate_email', + self.gf('django.db.models.fields.EmailField')(default='', max_length=255, blank=True), + keep_default=False) + + def backwards(self, orm): + db.delete_column('mirrors_mirror', 'alternate_email') + + models = { + 'mirrors.mirror': { + 'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + 'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) + }, + 'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) + }, + 'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index 8c2bd7fc..9a545b51 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -20,6 +20,7 @@ class Mirror(models.Model): upstream = models.ForeignKey('self', null=True, on_delete=models.SET_NULL) country = CountryField(blank=True, db_index=True) admin_email = models.EmailField(max_length=255, blank=True) + alternate_email = models.EmailField(max_length=255, blank=True) public = models.BooleanField(default=True) active = models.BooleanField(default=True) isos = models.BooleanField("ISOs", default=True) -- cgit v1.2.3-2-g168b From a87da032cb6b5b84624e4205b5f8b7cab37249cd Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 8 Jul 2012 20:36:51 -0500 Subject: Handle HTTPException being thrown in mirrorcheck Managed to see this bubble up today when running the mirrorcheck command on a less than ideal connection that was experiencing timeouts at the wrong time. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index ae89d5e0..3d431796 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -11,6 +11,7 @@ Usage: ./manage.py mirrorcheck from collections import deque from datetime import datetime +from httplib import HTTPException import logging import os from optparse import make_option @@ -115,6 +116,11 @@ def check_mirror_url(mirror_url, timeout): elif isinstance(e.reason, socket.error): log.error = e.reason.args[1] logger.debug("failed: %s, %s", url, log.error) + except HTTPException as e: + # e.g., BadStatusLine + log.is_success = False + log.error = "Exception in processing HTTP request." + logger.debug("failed: %s, %s", url, log.error) except socket.timeout as e: log.is_success = False log.error = "Connection timed out." -- cgit v1.2.3-2-g168b From 0f3c894e7a0f573fa0198459150f387c3a7f23ae Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 8 Jul 2012 20:38:01 -0500 Subject: Don't include StdDev on sqlite3 mirror status query Because this function isn't shipped by default, it makes more sense to just omit it completely from the query we do to build the tables on this page when in development. Substitute 0.0 for the value so the rest of the calculations and display work as expected. Signed-off-by: Dan McGee --- mirrors/utils.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index 2014411d..9aa8e0f5 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -3,7 +3,7 @@ from datetime import timedelta from django.db.models import Avg, Count, Max, Min, StdDev from django_countries.fields import Country -from main.utils import cache_function, utc_now +from main.utils import cache_function, utc_now, database_vendor from .models import MirrorLog, MirrorProtocol, MirrorUrl @@ -40,8 +40,11 @@ def get_mirror_statuses(cutoff=default_cutoff): success_count=Count('logs__duration'), last_sync=Max('logs__last_sync'), last_check=Max('logs__check_time'), - duration_avg=Avg('logs__duration'), - duration_stddev=StdDev('logs__duration')) + duration_avg=Avg('logs__duration')) + + vendor = database_vendor(MirrorUrl) + if vendor != 'sqlite': + urls.annotate(duration_stddev=StdDev('logs__duration')) # The Django ORM makes it really hard to get actual average delay in the # above query, so run a seperate query for it and we will process the @@ -70,6 +73,9 @@ def get_mirror_statuses(cutoff=default_cutoff): check_frequency = None for url in urls: + # fake the standard deviation for local testing setups + if vendor == 'sqlite': + setattr(url, 'duration_stddev', 0.0) annotate_url(url, delays) return { -- cgit v1.2.3-2-g168b From 3c4ceb16331b37fd334dc9682d4cde6430838942 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 8 Jul 2012 20:56:28 -0500 Subject: mirrorcheck: Don't use bulk_create on sqlite3 It isn't worth it, as we run into the 999 max SQL statement variables issue when using it on any significant amount of mirrors. Since this is just a development database setup, and it isn't a command we need to run especially fast, we can ditch it. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 3d431796..7a133cbf 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -30,7 +30,7 @@ import urllib2 from django.core.management.base import NoArgsCommand from django.db import transaction -from main.utils import utc_now +from main.utils import utc_now, database_vendor from mirrors.models import MirrorUrl, MirrorLog logging.basicConfig( @@ -207,7 +207,11 @@ class MirrorCheckPool(object): logger.debug("joining on all threads") self.tasks.join() logger.debug("processing %d log entries", len(self.logs)) - MirrorLog.objects.bulk_create(self.logs) + if database_vendor(MirrorLog, mode='write') == 'sqlite': + for log in self.logs: + log.save(force_insert=True) + else: + MirrorLog.objects.bulk_create(self.logs) logger.debug("log entries saved") # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From 9c7350650e66b5eb6228778e335a160be5ea7f98 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 8 Jul 2012 21:53:35 -0500 Subject: Correctly reassign queryset with added annotation in mirror status This was a dumb oversight on my part in commit 0f3c894e7a0. Signed-off-by: Dan McGee --- mirrors/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index 9aa8e0f5..f2c98ee0 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -44,7 +44,7 @@ def get_mirror_statuses(cutoff=default_cutoff): vendor = database_vendor(MirrorUrl) if vendor != 'sqlite': - urls.annotate(duration_stddev=StdDev('logs__duration')) + urls = urls.annotate(duration_stddev=StdDev('logs__duration')) # The Django ORM makes it really hard to get actual average delay in the # above query, so run a seperate query for it and we will process the -- cgit v1.2.3-2-g168b From c0bf9e20660cfae7ea8994472555bba23398b598 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 24 Jul 2012 09:19:48 -0500 Subject: Remove custom utc_now() function, use django.utils.timezone.now() This was around from the time when we handled timezones sanely and Django did not; now that we are on 1.4 we no longer need our own code to handle this. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 7 ++++--- mirrors/utils.py | 17 +++++++++-------- 2 files changed, 13 insertions(+), 11 deletions(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 7a133cbf..e09ea680 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -29,8 +29,9 @@ import urllib2 from django.core.management.base import NoArgsCommand from django.db import transaction +from django.utils.timezone import now -from main.utils import utc_now, database_vendor +from main.utils import database_vendor from mirrors.models import MirrorUrl, MirrorLog logging.basicConfig( @@ -83,7 +84,7 @@ def parse_lastsync(log, data): def check_mirror_url(mirror_url, timeout): url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) - log = MirrorLog(url=mirror_url, check_time=utc_now()) + log = MirrorLog(url=mirror_url, check_time=now()) headers = {'User-Agent': 'archweb/1.0'} req = urllib2.Request(url, None, headers) try: @@ -136,7 +137,7 @@ def check_mirror_url(mirror_url, timeout): def check_rsync_url(mirror_url, timeout): url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) - log = MirrorLog(url=mirror_url, check_time=utc_now()) + log = MirrorLog(url=mirror_url, check_time=now()) tempdir = tempfile.mkdtemp() lastsync_path = os.path.join(tempdir, 'lastsync') diff --git a/mirrors/utils.py b/mirrors/utils.py index f2c98ee0..bf030d39 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -1,13 +1,14 @@ from datetime import timedelta from django.db.models import Avg, Count, Max, Min, StdDev +from django.utils.timezone import now from django_countries.fields import Country -from main.utils import cache_function, utc_now, database_vendor +from main.utils import cache_function, database_vendor from .models import MirrorLog, MirrorProtocol, MirrorUrl -default_cutoff = timedelta(hours=24) +DEFAULT_CUTOFF = timedelta(hours=24) def annotate_url(url, delays): '''Given a MirrorURL object, add a few more attributes to it regarding @@ -30,8 +31,8 @@ def annotate_url(url, delays): @cache_function(123) -def get_mirror_statuses(cutoff=default_cutoff): - cutoff_time = utc_now() - cutoff +def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): + cutoff_time = now() - cutoff # I swear, this actually has decent performance... urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( mirror__active=True, mirror__public=True, @@ -88,8 +89,8 @@ def get_mirror_statuses(cutoff=default_cutoff): @cache_function(117) -def get_mirror_errors(cutoff=default_cutoff): - cutoff_time = utc_now() - cutoff +def get_mirror_errors(cutoff=DEFAULT_CUTOFF): + cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( is_success=False, check_time__gte=cutoff_time, url__mirror__active=True, url__mirror__public=True).values( @@ -105,11 +106,11 @@ def get_mirror_errors(cutoff=default_cutoff): @cache_function(295) -def get_mirror_url_for_download(cutoff=default_cutoff): +def get_mirror_url_for_download(cutoff=DEFAULT_CUTOFF): '''Find a good mirror URL to use for package downloads. If we have mirror status data available, it is used to determine a good choice by looking at the last batch of status rows.''' - cutoff_time = utc_now() - cutoff + cutoff_time = now() - cutoff status_data = MirrorLog.objects.filter( check_time__gte=cutoff_time).aggregate( Max('check_time'), Max('last_sync')) -- cgit v1.2.3-2-g168b From 76c37ce3acc7a4af0271c7535d4a33042f7749b5 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 24 Jul 2012 09:35:55 -0500 Subject: Replace deprecated direct_to_template() with render() shortcut Now that Django actually provides a concise way to use a RequestContext object without instantiating it, we can use that rather than the old function-based generic view that worked well to do the same. Additionally, these function-based generic views will be gone in Django 1.5, so might as well make the move now. Signed-off-by: Dan McGee --- mirrors/views.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) (limited to 'mirrors') diff --git a/mirrors/views.py b/mirrors/views.py index 8f092be7..400c084d 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -8,9 +8,8 @@ from django.forms.widgets import CheckboxSelectMultiple from django.core.serializers.json import DjangoJSONEncoder from django.db.models import Q from django.http import Http404, HttpResponse -from django.shortcuts import get_object_or_404 +from django.shortcuts import get_object_or_404, render from django.views.decorators.csrf import csrf_exempt -from django.views.generic.simple import direct_to_template from django_countries.countries import COUNTRIES from .models import Mirror, MirrorUrl, MirrorProtocol, TIER_CHOICES @@ -76,7 +75,7 @@ def generate_mirrorlist(request): else: form = MirrorlistForm() - return direct_to_template(request, 'mirrors/mirrorlist_generate.html', + return render(request, 'mirrors/mirrorlist_generate.html', {'mirrorlist_form': form}) @@ -142,10 +141,10 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, urls = status_filter(urls) template = 'mirrors/mirrorlist_status.txt' - return direct_to_template(request, template, { - 'mirror_urls': urls, - }, - mimetype='text/plain') + context = { + 'mirror_urls': urls, + } + return render(request, template, context, content_type='text/plain') def find_mirrors_simple(request, protocol): @@ -161,7 +160,7 @@ def mirrors(request): mirror_list = Mirror.objects.select_related().order_by('tier', 'country') if not request.user.is_authenticated(): mirror_list = mirror_list.filter(public=True, active=True) - return direct_to_template(request, 'mirrors/mirrors.html', + return render(request, 'mirrors/mirrors.html', {'mirror_list': mirror_list}) @@ -180,7 +179,7 @@ def mirror_details(request, name): all_urls = set(checked_urls).union(all_urls) all_urls = sorted(all_urls, key=attrgetter('url')) - return direct_to_template(request, 'mirrors/mirror_details.html', + return render(request, 'mirrors/mirror_details.html', {'mirror': mirror, 'urls': all_urls}) @@ -217,7 +216,7 @@ def status(request, tier=None): 'error_logs': error_logs, 'tier': tier, }) - return direct_to_template(request, 'mirrors/status.html', context) + return render(request, 'mirrors/status.html', context) class MirrorStatusJSONEncoder(DjangoJSONEncoder): -- cgit v1.2.3-2-g168b From 686942b8788fa43031b3999ac00d60baadc82f53 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 31 Jul 2012 19:56:49 -0500 Subject: Declare 'enums' at class scope Signed-off-by: Dan McGee --- mirrors/models.py | 15 +++++++-------- mirrors/views.py | 4 ++-- 2 files changed, 9 insertions(+), 10 deletions(-) (limited to 'mirrors') diff --git a/mirrors/models.py b/mirrors/models.py index 9a545b51..06b483d5 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -6,15 +6,14 @@ from django.core.exceptions import ValidationError from django_countries import CountryField -TIER_CHOICES = ( - (0, 'Tier 0'), - (1, 'Tier 1'), - (2, 'Tier 2'), - (-1, 'Untiered'), -) - - class Mirror(models.Model): + TIER_CHOICES = ( + (0, 'Tier 0'), + (1, 'Tier 1'), + (2, 'Tier 2'), + (-1, 'Untiered'), + ) + name = models.CharField(max_length=255, unique=True) tier = models.SmallIntegerField(default=2, choices=TIER_CHOICES) upstream = models.ForeignKey('self', null=True, on_delete=models.SET_NULL) diff --git a/mirrors/views.py b/mirrors/views.py index 400c084d..2c2577f4 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -12,7 +12,7 @@ from django.shortcuts import get_object_or_404, render from django.views.decorators.csrf import csrf_exempt from django_countries.countries import COUNTRIES -from .models import Mirror, MirrorUrl, MirrorProtocol, TIER_CHOICES +from .models import Mirror, MirrorUrl, MirrorProtocol from .utils import get_mirror_statuses, get_mirror_errors COUNTRY_LOOKUP = dict(COUNTRIES) @@ -186,7 +186,7 @@ def mirror_details(request, name): def status(request, tier=None): if tier is not None: tier = int(tier) - if tier not in [t[0] for t in TIER_CHOICES]: + if tier not in [t[0] for t in Mirror.TIER_CHOICES]: raise Http404 bad_timedelta = timedelta(days=3) status_info = get_mirror_statuses() -- cgit v1.2.3-2-g168b From b7a03d89d126989bf53005404759482e17163991 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 7 Aug 2012 23:22:18 -0500 Subject: Push more default values down into the database This makes it easier to do manual manipulation/insertion/etc. at the database level, as well as just making things act more sane from an overall software stack perspective. Signed-off-by: Dan McGee --- mirrors/migrations/0004_auto__add_field_mirrorprotocol_is_download.py | 2 +- ...uto__add_field_mirrorurl_has_ipv4__add_field_mirrorurl_has_ipv6.py | 4 ++-- mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py | 2 +- mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) (limited to 'mirrors') diff --git a/mirrors/migrations/0004_auto__add_field_mirrorprotocol_is_download.py b/mirrors/migrations/0004_auto__add_field_mirrorprotocol_is_download.py index 5e44d211..0506e2cd 100644 --- a/mirrors/migrations/0004_auto__add_field_mirrorprotocol_is_download.py +++ b/mirrors/migrations/0004_auto__add_field_mirrorprotocol_is_download.py @@ -7,7 +7,7 @@ from django.db import models class Migration(SchemaMigration): def forwards(self, orm): - db.add_column('mirrors_mirrorprotocol', 'is_download', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=False) + db.add_column('mirrors_mirrorprotocol', 'is_download', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=True) def backwards(self, orm): db.delete_column('mirrors_mirrorprotocol', 'is_download') diff --git a/mirrors/migrations/0006_auto__add_field_mirrorurl_has_ipv4__add_field_mirrorurl_has_ipv6.py b/mirrors/migrations/0006_auto__add_field_mirrorurl_has_ipv4__add_field_mirrorurl_has_ipv6.py index a5e34589..5a40207d 100644 --- a/mirrors/migrations/0006_auto__add_field_mirrorurl_has_ipv4__add_field_mirrorurl_has_ipv6.py +++ b/mirrors/migrations/0006_auto__add_field_mirrorurl_has_ipv4__add_field_mirrorurl_has_ipv6.py @@ -7,8 +7,8 @@ from django.db import models class Migration(SchemaMigration): def forwards(self, orm): - db.add_column('mirrors_mirrorurl', 'has_ipv4', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=False) - db.add_column('mirrors_mirrorurl', 'has_ipv6', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=False) + db.add_column('mirrors_mirrorurl', 'has_ipv4', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=True) + db.add_column('mirrors_mirrorurl', 'has_ipv6', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=True) def backwards(self, orm): db.delete_column('mirrors_mirrorurl', 'has_ipv4') diff --git a/mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py b/mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py index d30c78c7..66e60090 100644 --- a/mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py +++ b/mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py @@ -6,7 +6,7 @@ from django.db import models class Migration(SchemaMigration): def forwards(self, orm): - db.add_column('mirrors_mirrorprotocol', 'default', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=False) + db.add_column('mirrors_mirrorprotocol', 'default', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=True) def backwards(self, orm): db.delete_column('mirrors_mirrorprotocol', 'default') diff --git a/mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py b/mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py index 2f76c099..60c4ec26 100644 --- a/mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py +++ b/mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py @@ -7,7 +7,7 @@ from django.db import models class Migration(SchemaMigration): def forwards(self, orm): - db.alter_column('mirrors_mirrorlog', 'error', self.gf('django.db.models.fields.TextField')()) + db.alter_column('mirrors_mirrorlog', 'error', self.gf('django.db.models.fields.TextField')(default='')) def backwards(self, orm): db.alter_column('mirrors_mirrorlog', 'error', self.gf('django.db.models.fields.CharField')(max_length=255)) -- cgit v1.2.3-2-g168b From e9c4985538c067a09a186967f77c5395fb60675b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 18 Sep 2012 22:10:54 -0500 Subject: Sort mirrorlist by English country name, not ISO code Fixes FS#31503. Signed-off-by: Dan McGee --- mirrors/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/views.py b/mirrors/views.py index 2c2577f4..11719223 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -134,7 +134,7 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, urls = qset if not use_status: - sort_key = attrgetter('real_country', 'mirror.name', 'url') + sort_key = attrgetter('real_country.name', 'mirror.name', 'url') urls = sorted(urls, key=sort_key) template = 'mirrors/mirrorlist.txt' else: -- cgit v1.2.3-2-g168b From f0b7e73de61c03a5018ed352605e6329611448d2 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 10 Oct 2012 20:17:55 -0500 Subject: Make mirror log time query a bit more efficient We don't need the full mirror log objects; we just need a very small subset of values from them here to do the required math and object building. Signed-off-by: Dan McGee --- mirrors/utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index bf030d39..0a32b766 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -50,12 +50,14 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): # The Django ORM makes it really hard to get actual average delay in the # above query, so run a seperate query for it and we will process the # results here. - times = MirrorLog.objects.filter(is_success=True, last_sync__isnull=False, + times = MirrorLog.objects.values_list( + 'url_id', 'check_time', 'last_sync').filter( + is_success=True, last_sync__isnull=False, check_time__gte=cutoff_time) delays = {} - for log in times: - delay = log.check_time - log.last_sync - delays.setdefault(log.url_id, []).append(delay) + for url_id, check_time, last_sync in times: + delay = check_time - last_sync + delays.setdefault(url_id, []).append(delay) if urls: last_check = max([u.last_check for u in urls]) -- cgit v1.2.3-2-g168b From 4ab5d6947795f1fef0d38601ec7ad3ca5f62173e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 10 Nov 2012 14:13:34 -0600 Subject: Add mirror extended status JSON view When asking for status for a single mirror, we can include logs from the past 24 hours in addition to the normal information we provide. This is slated for usage by a frontend graph still to come, similar to those on the NTP pool website. Signed-off-by: Dan McGee --- mirrors/urls.py | 1 + mirrors/views.py | 37 ++++++++++++++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 3 deletions(-) (limited to 'mirrors') diff --git a/mirrors/urls.py b/mirrors/urls.py index bb4eb969..857e99e2 100644 --- a/mirrors/urls.py +++ b/mirrors/urls.py @@ -6,6 +6,7 @@ urlpatterns = patterns('mirrors.views', (r'^status/json/$', 'status_json', {}, 'mirror-status-json'), (r'^status/tier/(?P\d+)/$', 'status', {}, 'mirror-status-tier'), (r'^(?P[\.\-\w]+)/$', 'mirror_details'), + (r'^(?P[\.\-\w]+)/json/$', 'mirror_details_json'), ) # vim: set ts=4 sw=4 et: diff --git a/mirrors/views.py b/mirrors/views.py index 11719223..cbd86611 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -9,10 +9,11 @@ from django.core.serializers.json import DjangoJSONEncoder from django.db.models import Q from django.http import Http404, HttpResponse from django.shortcuts import get_object_or_404, render +from django.utils.timezone import now from django.views.decorators.csrf import csrf_exempt from django_countries.countries import COUNTRIES -from .models import Mirror, MirrorUrl, MirrorProtocol +from .models import Mirror, MirrorUrl, MirrorProtocol, MirrorLog from .utils import get_mirror_statuses, get_mirror_errors COUNTRY_LOOKUP = dict(COUNTRIES) @@ -183,6 +184,19 @@ def mirror_details(request, name): {'mirror': mirror, 'urls': all_urls}) +def mirror_details_json(request, name): + mirror = get_object_or_404(Mirror, name=name) + status_info = get_mirror_statuses() + data = status_info.copy() + data['version'] = 3 + # include only URLs for this particular mirror + data['urls'] = [url for url in data['urls'] if url.mirror_id == mirror.id] + to_json = json.dumps(data, ensure_ascii=False, + cls=ExtendedMirrorStatusJSONEncoder) + response = HttpResponse(to_json, mimetype='application/json') + return response + + def status(request, tier=None): if tier is not None: tier = int(tier) @@ -222,8 +236,8 @@ def status(request, tier=None): class MirrorStatusJSONEncoder(DjangoJSONEncoder): '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl serialization. The base class takes care of datetime.datetime types.''' - url_attributes = ['url', 'protocol', 'last_sync', 'completion_pct', - 'delay', 'duration_avg', 'duration_stddev', 'score'] + url_attributes = ('url', 'protocol', 'last_sync', 'completion_pct', + 'delay', 'duration_avg', 'duration_stddev', 'score') def default(self, obj): if isinstance(obj, timedelta): @@ -245,6 +259,23 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder): return super(MirrorStatusJSONEncoder, self).default(obj) +class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): + '''Adds URL check history information.''' + log_attributes = ('check_time', 'last_sync', 'duration', 'is_success') + + def default(self, obj): + if isinstance(obj, MirrorUrl): + data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj) + cutoff = now() - timedelta(hours=24) + data['logs'] = obj.logs.filter(check_time__gte=cutoff) + return data + if isinstance(obj, MirrorLog): + data = dict((attr, getattr(obj, attr)) + for attr in self.log_attributes) + return data + return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) + + def status_json(request): status_info = get_mirror_statuses() data = status_info.copy() -- cgit v1.2.3-2-g168b From 86102c6e645451c03e3e576060eba7f93350bf6b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 10 Nov 2012 14:19:23 -0600 Subject: Allow filtering retrieved mirror statuses by mirror_id When we don't need them all, no need to fetch them all. Let the database do the work for us, hopefully. Signed-off-by: Dan McGee --- mirrors/utils.py | 19 +++++++++++++++---- mirrors/views.py | 8 ++++---- 2 files changed, 19 insertions(+), 8 deletions(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index 0a32b766..ba027c99 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -31,7 +31,7 @@ def annotate_url(url, delays): @cache_function(123) -def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): +def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): cutoff_time = now() - cutoff # I swear, this actually has decent performance... urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( @@ -43,6 +43,9 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): last_check=Max('logs__check_time'), duration_avg=Avg('logs__duration')) + if mirror_ids: + urls = urls.filter(mirror_id__in=mirror_ids) + vendor = database_vendor(MirrorUrl) if vendor != 'sqlite': urls = urls.annotate(duration_stddev=StdDev('logs__duration')) @@ -54,6 +57,8 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): 'url_id', 'check_time', 'last_sync').filter( is_success=True, last_sync__isnull=False, check_time__gte=cutoff_time) + if mirror_ids: + times = times.filter(url__mirror_id__in=mirror_ids) delays = {} for url_id, check_time, last_sync in times: delay = check_time - last_sync @@ -62,8 +67,10 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): if urls: last_check = max([u.last_check for u in urls]) num_checks = max([u.check_count for u in urls]) - check_info = MirrorLog.objects.filter( - check_time__gte=cutoff_time).aggregate( + check_info = MirrorLog.objects.filter(check_time__gte=cutoff_time) + if mirror_ids: + check_info = check_info.filter(url__mirror_id__in=mirror_ids) + check_info = check_info.aggregate( mn=Min('check_time'), mx=Max('check_time')) if num_checks > 1: check_frequency = (check_info['mx'] - check_info['mn']) \ @@ -91,7 +98,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF): @cache_function(117) -def get_mirror_errors(cutoff=DEFAULT_CUTOFF): +def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None): cutoff_time = now() - cutoff errors = MirrorLog.objects.filter( is_success=False, check_time__gte=cutoff_time, @@ -100,6 +107,10 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF): 'url__mirror__country', 'url__mirror__tier', 'error').annotate( error_count=Count('error'), last_occurred=Max('check_time') ).order_by('-last_occurred', '-error_count') + + if mirror_ids: + urls = urls.filter(mirror_id__in=mirror_ids) + errors = list(errors) for err in errors: ctry_code = err['url__country'] or err['url__mirror__country'] diff --git a/mirrors/views.py b/mirrors/views.py index cbd86611..4b9721dc 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -14,7 +14,7 @@ from django.views.decorators.csrf import csrf_exempt from django_countries.countries import COUNTRIES from .models import Mirror, MirrorUrl, MirrorProtocol, MirrorLog -from .utils import get_mirror_statuses, get_mirror_errors +from .utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF COUNTRY_LOOKUP = dict(COUNTRIES) @@ -171,7 +171,7 @@ def mirror_details(request, name): (not mirror.public or not mirror.active): raise Http404 - status_info = get_mirror_statuses() + status_info = get_mirror_statuses(mirror_ids=[mirror.id]) checked_urls = [url for url in status_info['urls'] \ if url.mirror_id == mirror.id] all_urls = mirror.urls.select_related('protocol') @@ -186,7 +186,7 @@ def mirror_details(request, name): def mirror_details_json(request, name): mirror = get_object_or_404(Mirror, name=name) - status_info = get_mirror_statuses() + status_info = get_mirror_statuses(mirror_ids=[mirror.id]) data = status_info.copy() data['version'] = 3 # include only URLs for this particular mirror @@ -266,7 +266,7 @@ class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): def default(self, obj): if isinstance(obj, MirrorUrl): data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj) - cutoff = now() - timedelta(hours=24) + cutoff = now() - DEFAULT_CUTOFF data['logs'] = obj.logs.filter(check_time__gte=cutoff) return data if isinstance(obj, MirrorLog): -- cgit v1.2.3-2-g168b From 07d2fc5d358992a52908cccbbca4a11d01e98da3 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 10 Nov 2012 15:41:09 -0600 Subject: Add initial version of mirror status chart Still have some hardcoded stuff to rip out and replace to make this a bit more dynamic on things like sizing, but for now, this is a great start. Signed-off-by: Dan McGee --- mirrors/static/mirror_status.js | 97 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 mirrors/static/mirror_status.js (limited to 'mirrors') diff --git a/mirrors/static/mirror_status.js b/mirrors/static/mirror_status.js new file mode 100644 index 00000000..1c510cee --- /dev/null +++ b/mirrors/static/mirror_status.js @@ -0,0 +1,97 @@ +function mirror_status(chart_id, data_url) { + d3.json(data_url, function(json) { + data = jQuery.map(json['urls'], + function(url, i) { + return jQuery.map(url['logs'], + function(log, j) { + return { + url: url['url'], + duration: log['duration'], + check_time: new Date(log['check_time']) + }; + }); + }); + + var margin = {top: 20, right: 20, bottom: 30, left: 40}, + width = 1200 - margin.left - margin.right, + height = 450 - margin.top - margin.bottom; + + var color = d3.scale.category20(), + x = d3.time.scale.utc().range([0, width]), + y = d3.scale.linear().range([height, 0]), + x_axis = d3.svg.axis().scale(x).orient("bottom"), + y_axis = d3.svg.axis().scale(y).orient("left"); + + var svg = d3.select(chart_id).append("svg") + .attr("width", width + margin.left + margin.right) + .attr("height", height + margin.top + margin.bottom) + .append("g") + .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); + + x.domain(d3.extent(data, function(d) { return d.check_time; })).nice(d3.time.hour); + y.domain(d3.extent(data, function(d) { return d.duration; })).nice(); + + svg.append("g") + .attr("class", "x axis") + .attr("transform", "translate(0," + height + ")") + .call(x_axis) + .append("text") + .attr("class", "label") + .attr("x", width) + .attr("y", -6) + .style("text-anchor", "end") + .text("Check Time (UTC)"); + + svg.append("g") + .attr("class", "y axis") + .call(y_axis) + .append("text") + .attr("class", "label") + .attr("transform", "rotate(-90)") + .attr("y", 6) + .attr("dy", ".71em") + .style("text-anchor", "end") + .text("Duration (seconds)"); + + svg.selectAll(".dot") + .data(data) + .enter() + .append("circle") + .attr("class", "dot") + .attr("r", 3.5) + .attr("cx", function(d) { return x(d.check_time); }) + .attr("cy", function(d) { return y(d.duration); }) + .style("fill", function(d) { return color(d.url); }); + + var legend = svg.selectAll(".legend") + .data(color.domain()) + .enter().append("g") + .attr("class", "legend") + .attr("transform", function(d, i) { return "translate(0," + i * 20 + ")"; }); + + legend.append("rect") + .attr("x", width - 18) + .attr("width", 18) + .attr("height", 18) + .style("fill", color); + + legend.append("text") + .attr("x", width - 24) + .attr("y", 9) + .attr("dy", ".35em") + .style("text-anchor", "end") + .text(function(d) { return d; }); + }); + + var resize_timeout = null; + var real_resize = function() { + resize_timeout = null; + /* TODO: implement resize */ + }; + jQuery(window).resize(function() { + if (resize_timeout) { + clearTimeout(resize_timeout); + } + resize_timeout = setTimeout(real_resize, 200); + }); +} -- cgit v1.2.3-2-g168b From aeeb4718e83cd2f82d94b1aa0c0ba36ba21a2b37 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 10 Nov 2012 16:12:15 -0600 Subject: Enable mirror status graph resizing Signed-off-by: Dan McGee --- mirrors/static/mirror_status.js | 43 +++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 15 deletions(-) (limited to 'mirrors') diff --git a/mirrors/static/mirror_status.js b/mirrors/static/mirror_status.js index 1c510cee..277d3c88 100644 --- a/mirrors/static/mirror_status.js +++ b/mirrors/static/mirror_status.js @@ -1,20 +1,10 @@ function mirror_status(chart_id, data_url) { - d3.json(data_url, function(json) { - data = jQuery.map(json['urls'], - function(url, i) { - return jQuery.map(url['logs'], - function(log, j) { - return { - url: url['url'], - duration: log['duration'], - check_time: new Date(log['check_time']) - }; - }); - }); + var jq_div = jQuery(chart_id); + var draw_graph = function(data) { var margin = {top: 20, right: 20, bottom: 30, left: 40}, - width = 1200 - margin.left - margin.right, - height = 450 - margin.top - margin.bottom; + width = jq_div.width() - margin.left - margin.right, + height = jq_div.height() - margin.top - margin.bottom; var color = d3.scale.category20(), x = d3.time.scale.utc().range([0, width]), @@ -22,6 +12,8 @@ function mirror_status(chart_id, data_url) { x_axis = d3.svg.axis().scale(x).orient("bottom"), y_axis = d3.svg.axis().scale(y).orient("left"); + /* remove any existing graph first if we are redrawing after resize */ + d3.select(chart_id).select("svg").remove(); var svg = d3.select(chart_id).append("svg") .attr("width", width + margin.left + margin.right) .attr("height", height + margin.top + margin.bottom) @@ -31,6 +23,7 @@ function mirror_status(chart_id, data_url) { x.domain(d3.extent(data, function(d) { return d.check_time; })).nice(d3.time.hour); y.domain(d3.extent(data, function(d) { return d.duration; })).nice(); + /* build the axis lines... */ svg.append("g") .attr("class", "x axis") .attr("transform", "translate(0," + height + ")") @@ -53,6 +46,7 @@ function mirror_status(chart_id, data_url) { .style("text-anchor", "end") .text("Duration (seconds)"); + /* ...then the points themselves. */ svg.selectAll(".dot") .data(data) .enter() @@ -63,6 +57,7 @@ function mirror_status(chart_id, data_url) { .attr("cy", function(d) { return y(d.duration); }) .style("fill", function(d) { return color(d.url); }); + /* add a legend for good measure */ var legend = svg.selectAll(".legend") .data(color.domain()) .enter().append("g") @@ -81,12 +76,30 @@ function mirror_status(chart_id, data_url) { .attr("dy", ".35em") .style("text-anchor", "end") .text(function(d) { return d; }); + }; + + /* invoke the data-fetch + first draw */ + var cached_data = null; + d3.json(data_url, function(json) { + cached_data = jQuery.map(json['urls'], + function(url, i) { + return jQuery.map(url['logs'], + function(log, j) { + return { + url: url['url'], + duration: log['duration'], + check_time: new Date(log['check_time']) + }; + }); + }); + draw_graph(cached_data); }); + /* then hook up a resize handler to redraw if necessary */ var resize_timeout = null; var real_resize = function() { resize_timeout = null; - /* TODO: implement resize */ + draw_graph(cached_data); }; jQuery(window).resize(function() { if (resize_timeout) { -- cgit v1.2.3-2-g168b From a358e132886972dc4e9f1f546e36a5f3a2218a39 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 10 Nov 2012 17:09:51 -0600 Subject: Mirror status graph, now with points AND lines We might have to tweak the interpolation method once we see this with real data, but for now it looks really pretty locally. Signed-off-by: Dan McGee --- mirrors/static/mirror_status.js | 58 +++++++++++++++++++++++++++++------------ mirrors/views.py | 3 ++- 2 files changed, 44 insertions(+), 17 deletions(-) (limited to 'mirrors') diff --git a/mirrors/static/mirror_status.js b/mirrors/static/mirror_status.js index 277d3c88..d107d7d1 100644 --- a/mirrors/static/mirror_status.js +++ b/mirrors/static/mirror_status.js @@ -6,7 +6,7 @@ function mirror_status(chart_id, data_url) { width = jq_div.width() - margin.left - margin.right, height = jq_div.height() - margin.top - margin.bottom; - var color = d3.scale.category20(), + var color = d3.scale.category10(), x = d3.time.scale.utc().range([0, width]), y = d3.scale.linear().range([height, 0]), x_axis = d3.svg.axis().scale(x).orient("bottom"), @@ -20,8 +20,14 @@ function mirror_status(chart_id, data_url) { .append("g") .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); - x.domain(d3.extent(data, function(d) { return d.check_time; })).nice(d3.time.hour); - y.domain(d3.extent(data, function(d) { return d.duration; })).nice(); + x.domain([ + d3.min(data, function(c) { return d3.min(c.logs, function(v) { return v.check_time; }); }), + d3.max(data, function(c) { return d3.max(c.logs, function(v) { return v.check_time; }); }) + ]); + y.domain([ + d3.min(data, function(c) { return d3.min(c.logs, function(v) { return v.duration; }); }), + d3.max(data, function(c) { return d3.max(c.logs, function(v) { return v.duration; }); }) + ]); /* build the axis lines... */ svg.append("g") @@ -46,12 +52,32 @@ function mirror_status(chart_id, data_url) { .style("text-anchor", "end") .text("Duration (seconds)"); - /* ...then the points themselves. */ - svg.selectAll(".dot") + var line = d3.svg.line() + .interpolate("basis") + .x(function(d) { return x(d.check_time); }) + .y(function(d) { return y(d.duration); }); + + /* ...then the points and lines between them. */ + var urls = svg.selectAll(".url") .data(data) .enter() + .append("g") + .attr("class", "url"); + + urls.append("path") + .attr("class", "url-line") + .attr("d", function(d) { return line(d.logs); }) + .style("stroke", function(d) { return color(d.url); }); + + urls.selectAll("circle") + .data(function(u) { + return jQuery.map(u.logs, function(l, i) { + return {url: u.url, check_time: l.check_time, duration: l.duration}; + }); + }) + .enter() .append("circle") - .attr("class", "dot") + .attr("class", "url-dot") .attr("r", 3.5) .attr("cx", function(d) { return x(d.check_time); }) .attr("cy", function(d) { return y(d.duration); }) @@ -81,16 +107,16 @@ function mirror_status(chart_id, data_url) { /* invoke the data-fetch + first draw */ var cached_data = null; d3.json(data_url, function(json) { - cached_data = jQuery.map(json['urls'], - function(url, i) { - return jQuery.map(url['logs'], - function(log, j) { - return { - url: url['url'], - duration: log['duration'], - check_time: new Date(log['check_time']) - }; - }); + cached_data = jQuery.map(json.urls, function(url, i) { + return { + url: url.url, + logs: jQuery.map(url.logs, function(log, j) { + return { + duration: log.duration, + check_time: new Date(log.check_time) + }; + }) + }; }); draw_graph(cached_data); }); diff --git a/mirrors/views.py b/mirrors/views.py index 4b9721dc..be01e919 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -267,7 +267,8 @@ class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): if isinstance(obj, MirrorUrl): data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj) cutoff = now() - DEFAULT_CUTOFF - data['logs'] = obj.logs.filter(check_time__gte=cutoff) + data['logs'] = obj.logs.filter( + check_time__gte=cutoff).order_by('check_time') return data if isinstance(obj, MirrorLog): data = dict((attr, getattr(obj, attr)) -- cgit v1.2.3-2-g168b From 923ebbb53abf1d77a2f21b76e88faa085251af78 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 10 Nov 2012 17:20:26 -0600 Subject: Re-add nice() calls for mirror status axes Signed-off-by: Dan McGee --- mirrors/static/mirror_status.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'mirrors') diff --git a/mirrors/static/mirror_status.js b/mirrors/static/mirror_status.js index d107d7d1..1c352a9f 100644 --- a/mirrors/static/mirror_status.js +++ b/mirrors/static/mirror_status.js @@ -23,11 +23,11 @@ function mirror_status(chart_id, data_url) { x.domain([ d3.min(data, function(c) { return d3.min(c.logs, function(v) { return v.check_time; }); }), d3.max(data, function(c) { return d3.max(c.logs, function(v) { return v.check_time; }); }) - ]); + ]).nice(d3.time.hour); y.domain([ d3.min(data, function(c) { return d3.min(c.logs, function(v) { return v.duration; }); }), d3.max(data, function(c) { return d3.max(c.logs, function(v) { return v.duration; }); }) - ]); + ]).nice(); /* build the axis lines... */ svg.append("g") -- cgit v1.2.3-2-g168b From e26d5722289bd2e972633891d8dac09296b0cbc4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 11 Nov 2012 14:55:37 -0600 Subject: Mirror graph tweaking after usage with real data * Clamp y-axis minimum to 0. * Don't plot `is_success == false` values. * Ensure URLs are sorted predictably. Signed-off-by: Dan McGee --- mirrors/static/mirror_status.js | 5 ++++- mirrors/utils.py | 3 ++- mirrors/views.py | 2 -- 3 files changed, 6 insertions(+), 4 deletions(-) (limited to 'mirrors') diff --git a/mirrors/static/mirror_status.js b/mirrors/static/mirror_status.js index 1c352a9f..decc8fb8 100644 --- a/mirrors/static/mirror_status.js +++ b/mirrors/static/mirror_status.js @@ -25,7 +25,7 @@ function mirror_status(chart_id, data_url) { d3.max(data, function(c) { return d3.max(c.logs, function(v) { return v.check_time; }); }) ]).nice(d3.time.hour); y.domain([ - d3.min(data, function(c) { return d3.min(c.logs, function(v) { return v.duration; }); }), + 0, d3.max(data, function(c) { return d3.max(c.logs, function(v) { return v.duration; }); }) ]).nice(); @@ -111,6 +111,9 @@ function mirror_status(chart_id, data_url) { return { url: url.url, logs: jQuery.map(url.logs, function(log, j) { + if (!log.is_success) { + return null; + } return { duration: log.duration, check_time: new Date(log.check_time) diff --git a/mirrors/utils.py b/mirrors/utils.py index ba027c99..85e4ee93 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -41,7 +41,8 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): success_count=Count('logs__duration'), last_sync=Max('logs__last_sync'), last_check=Max('logs__check_time'), - duration_avg=Avg('logs__duration')) + duration_avg=Avg('logs__duration')).order_by( + 'mirror', 'url') if mirror_ids: urls = urls.filter(mirror_id__in=mirror_ids) diff --git a/mirrors/views.py b/mirrors/views.py index be01e919..5e374b4d 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -189,8 +189,6 @@ def mirror_details_json(request, name): status_info = get_mirror_statuses(mirror_ids=[mirror.id]) data = status_info.copy() data['version'] = 3 - # include only URLs for this particular mirror - data['urls'] = [url for url in data['urls'] if url.mirror_id == mirror.id] to_json = json.dumps(data, ensure_ascii=False, cls=ExtendedMirrorStatusJSONEncoder) response = HttpResponse(to_json, mimetype='application/json') -- cgit v1.2.3-2-g168b From 99bfdda5f257107396179694b7c56aad8e5e6701 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 11 Nov 2012 16:40:21 -0600 Subject: Add title to mirror status data points Signed-off-by: Dan McGee --- mirrors/static/mirror_status.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/static/mirror_status.js b/mirrors/static/mirror_status.js index decc8fb8..8ec85c40 100644 --- a/mirrors/static/mirror_status.js +++ b/mirrors/static/mirror_status.js @@ -81,7 +81,9 @@ function mirror_status(chart_id, data_url) { .attr("r", 3.5) .attr("cx", function(d) { return x(d.check_time); }) .attr("cy", function(d) { return y(d.duration); }) - .style("fill", function(d) { return color(d.url); }); + .style("fill", function(d) { return color(d.url); }) + .append("title") + .text(function(d) { return d.url + "\n" + d.duration.toFixed(3) + " secs\n" + d.check_time.toUTCString(); }); /* add a legend for good measure */ var legend = svg.selectAll(".legend") -- cgit v1.2.3-2-g168b From 55d2d3ae51e1d0aa0927d682b45a3500588ed07b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 12 Nov 2012 09:41:28 -0600 Subject: Add get_latest_by to MirrorLog Meta class Signed-off-by: Dan McGee --- mirrors/models.py | 1 + 1 file changed, 1 insertion(+) (limited to 'mirrors') diff --git a/mirrors/models.py b/mirrors/models.py index 06b483d5..384668b8 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -131,5 +131,6 @@ class MirrorLog(models.Model): class Meta: verbose_name = 'mirror check log' + get_latest_by = 'check_time' # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From 92837c93acc66056391dd0b98515b89f8fc49691 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 12 Nov 2012 21:37:08 -0600 Subject: Prefetch the available protocols on the mirror overview page Otherwise we are doing one query per mirror, which at this point is over 100 separate queries. Signed-off-by: Dan McGee --- mirrors/models.py | 5 ----- mirrors/views.py | 8 ++++++++ 2 files changed, 8 insertions(+), 5 deletions(-) (limited to 'mirrors') diff --git a/mirrors/models.py b/mirrors/models.py index 384668b8..0179d5bf 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -33,11 +33,6 @@ class Mirror(models.Model): def __unicode__(self): return self.name - def supported_protocols(self): - protocols = MirrorProtocol.objects.filter( - urls__mirror=self).order_by('protocol').distinct() - return sorted(protocols) - def downstream(self): return Mirror.objects.filter(upstream=self).order_by('name') diff --git a/mirrors/views.py b/mirrors/views.py index 5e374b4d..2e1e83b6 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -159,8 +159,16 @@ def find_mirrors_simple(request, protocol): def mirrors(request): mirror_list = Mirror.objects.select_related().order_by('tier', 'country') + protos = MirrorUrl.objects.values_list( + 'mirror_id', 'protocol__protocol').order_by( + 'mirror__id', 'protocol__protocol') if not request.user.is_authenticated(): mirror_list = mirror_list.filter(public=True, active=True) + protos = protos.filter(mirror__public=True, mirror__active=True) + protos = dict((k, list(v)) for k, v in groupby(protos, key=itemgetter(0))) + for mirror in mirror_list: + items = protos.get(mirror.id, []) + mirror.protocols = [item[1] for item in items] return render(request, 'mirrors/mirrors.html', {'mirror_list': mirror_list}) -- cgit v1.2.3-2-g168b From a2cfa7edbbed8edb1ad4d3391c6edb055c13de1b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 16 Nov 2012 15:39:56 -0600 Subject: Optimize mirror status data fetching Now that we have as many mirror URLs as we do, we can do a better job fetching and aggregating this data. The prior method resulted in a rather unwieldy query being pushed down to the database with a horrendously long GROUP BY clause. Instead of trying to group by everything at once so we can retrieve mirror URL info at the same time, separate the two queries- one for getting URL performance data, one for the qualitative data. The impetus behind fixing this is the PostgreSQL slow query log in production; this currently shows up the most of any queries we run in the system. Signed-off-by: Dan McGee --- mirrors/utils.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index 85e4ee93..07a7138f 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -33,23 +33,26 @@ def annotate_url(url, delays): @cache_function(123) def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): cutoff_time = now() - cutoff - # I swear, this actually has decent performance... - urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( + url_data = MirrorUrl.objects.values('id', 'mirror_id').filter( mirror__active=True, mirror__public=True, logs__check_time__gte=cutoff_time).annotate( check_count=Count('logs'), success_count=Count('logs__duration'), last_sync=Max('logs__last_sync'), last_check=Max('logs__check_time'), - duration_avg=Avg('logs__duration')).order_by( - 'mirror', 'url') - - if mirror_ids: - urls = urls.filter(mirror_id__in=mirror_ids) + duration_avg=Avg('logs__duration')) vendor = database_vendor(MirrorUrl) if vendor != 'sqlite': - urls = urls.annotate(duration_stddev=StdDev('logs__duration')) + url_data = url_data.annotate(duration_stddev=StdDev('logs__duration')) + + urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( + mirror__active=True, mirror__public=True, + logs__check_time__gte=cutoff_time).order_by('mirror__id', 'url') + + if mirror_ids: + url_data = url_data.filter(mirror_id__in=mirror_ids) + urls = urls.filter(mirror_id__in=mirror_ids) # The Django ORM makes it really hard to get actual average delay in the # above query, so run a seperate query for it and we will process the @@ -66,6 +69,11 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): delays.setdefault(url_id, []).append(delay) if urls: + url_data = dict((item['id'], item) for item in url_data) + for url in urls: + for k, v in url_data.get(url.id, {}).items(): + if k not in ('id', 'mirror_id'): + setattr(url, k, v) last_check = max([u.last_check for u in urls]) num_checks = max([u.check_count for u in urls]) check_info = MirrorLog.objects.filter(check_time__gte=cutoff_time) -- cgit v1.2.3-2-g168b From 6dd4d54bb0adbbb0f8c2b1beaa92b7a58971cf88 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 16 Nov 2012 16:20:11 -0600 Subject: Use Python 2.7 dictionary comprehension syntax Rather than the old idiom of dict((k, v) for <> in <>). Signed-off-by: Dan McGee --- mirrors/views.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) (limited to 'mirrors') diff --git a/mirrors/views.py b/mirrors/views.py index 2e1e83b6..d0ce0a97 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -94,7 +94,7 @@ def default_protocol_filter(original_urls): def status_filter(original_urls): status_info = get_mirror_statuses() - scores = dict((u.id, u.score) for u in status_info['urls']) + scores = {u.id: u.score for u in status_info['urls']} urls = [] for u in original_urls: u.score = scores.get(u.id, None) @@ -165,7 +165,7 @@ def mirrors(request): if not request.user.is_authenticated(): mirror_list = mirror_list.filter(public=True, active=True) protos = protos.filter(mirror__public=True, mirror__active=True) - protos = dict((k, list(v)) for k, v in groupby(protos, key=itemgetter(0))) + protos = {k: list(v) for k, v in groupby(protos, key=itemgetter(0))} for mirror in mirror_list: items = protos.get(mirror.id, []) mirror.protocols = [item[1] for item in items] @@ -253,8 +253,7 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder): # mainly for queryset serialization return list(obj) if isinstance(obj, MirrorUrl): - data = dict((attr, getattr(obj, attr)) - for attr in self.url_attributes) + data = {attr: getattr(obj, attr) for attr in self.url_attributes} # get any override on the country attribute first country = obj.real_country data['country'] = unicode(country.name) @@ -277,9 +276,7 @@ class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): check_time__gte=cutoff).order_by('check_time') return data if isinstance(obj, MirrorLog): - data = dict((attr, getattr(obj, attr)) - for attr in self.log_attributes) - return data + return {attr: getattr(obj, attr) for attr in self.log_attributes} return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) -- cgit v1.2.3-2-g168b From f0f6f7235a62186c1cae9c79036dde5d8821373d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 19 Nov 2012 08:10:21 -0600 Subject: Fix mirror URL duplication in status view We need to ensure we don't duplicate URLs in the status view, so add a distinct() call back in to the queryset when it was inadvertently dropped in commit a2cfa7edbb. This negates a lot of the performance gains we had, unfortunately, so it looks like a nested subquery might be more efficient. Disappointing the planner can't do this for us. Signed-off-by: Dan McGee --- mirrors/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index 07a7138f..a62c7f05 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -48,7 +48,8 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( mirror__active=True, mirror__public=True, - logs__check_time__gte=cutoff_time).order_by('mirror__id', 'url') + logs__check_time__gte=cutoff_time).distinct().order_by( + 'mirror__id', 'url') if mirror_ids: url_data = url_data.filter(mirror_id__in=mirror_ids) -- cgit v1.2.3-2-g168b From 2b68963ad1049f4b0198bed6da16e385aedc119e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 27 Dec 2012 16:37:49 -0600 Subject: Ensure mirror protocols are distinct Signed-off-by: Dan McGee --- mirrors/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/views.py b/mirrors/views.py index d0ce0a97..22da631a 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -161,7 +161,7 @@ def mirrors(request): mirror_list = Mirror.objects.select_related().order_by('tier', 'country') protos = MirrorUrl.objects.values_list( 'mirror_id', 'protocol__protocol').order_by( - 'mirror__id', 'protocol__protocol') + 'mirror__id', 'protocol__protocol').distinct() if not request.user.is_authenticated(): mirror_list = mirror_list.filter(public=True, active=True) protos = protos.filter(mirror__public=True, mirror__active=True) -- cgit v1.2.3-2-g168b From bec73c7a37c07821f145dbcf11435d4f2b94a149 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 12 Jan 2013 15:56:36 -0600 Subject: Round two of mirror status query improvements This seems to generate much more performant queries at the database level than what we were previously doing, and also doesn't show duplicate rows. Signed-off-by: Dan McGee --- mirrors/utils.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index a62c7f05..1d560021 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -33,9 +33,16 @@ def annotate_url(url, delays): @cache_function(123) def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): cutoff_time = now() - cutoff - url_data = MirrorUrl.objects.values('id', 'mirror_id').filter( + + valid_urls = MirrorUrl.objects.filter( mirror__active=True, mirror__public=True, - logs__check_time__gte=cutoff_time).annotate( + logs__check_time__gte=cutoff_time).distinct() + + if mirror_ids: + valid_urls = valid_urls.filter(mirror_id__in=mirror_ids) + + url_data = MirrorUrl.objects.values('id', 'mirror_id').filter( + id__in=valid_urls, logs__check_time__gte=cutoff_time).annotate( check_count=Count('logs'), success_count=Count('logs__duration'), last_sync=Max('logs__last_sync'), @@ -47,13 +54,7 @@ def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_ids=None): url_data = url_data.annotate(duration_stddev=StdDev('logs__duration')) urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter( - mirror__active=True, mirror__public=True, - logs__check_time__gte=cutoff_time).distinct().order_by( - 'mirror__id', 'url') - - if mirror_ids: - url_data = url_data.filter(mirror_id__in=mirror_ids) - urls = urls.filter(mirror_id__in=mirror_ids) + id__in=valid_urls).order_by('mirror__id', 'url') # The Django ORM makes it really hard to get actual average delay in the # above query, so run a seperate query for it and we will process the -- cgit v1.2.3-2-g168b From 66850026ca934e5a09238e9033c541cdc5085a42 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 13 Jan 2013 22:34:33 -0600 Subject: Use content_type and not mimetype on HttpResponse() Bug #16519 in Django deprecates mimetype, so update our code accordingly. Signed-off-by: Dan McGee --- mirrors/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'mirrors') diff --git a/mirrors/views.py b/mirrors/views.py index 22da631a..d3867802 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -199,7 +199,7 @@ def mirror_details_json(request, name): data['version'] = 3 to_json = json.dumps(data, ensure_ascii=False, cls=ExtendedMirrorStatusJSONEncoder) - response = HttpResponse(to_json, mimetype='application/json') + response = HttpResponse(to_json, content_type='application/json') return response @@ -285,7 +285,7 @@ def status_json(request): data = status_info.copy() data['version'] = 3 to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder) - response = HttpResponse(to_json, mimetype='application/json') + response = HttpResponse(to_json, content_type='application/json') return response # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From 5b8b6991b0b9f8174f153fe4b46376451b222cc1 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 14 Jan 2013 00:34:49 -0600 Subject: Add migration to move country data down to the URL level Rather than have the weird indirection we need now to find the right country for URLs, just always store it on the URL. Signed-off-by: Dan McGee --- .../migrations/0019_move_country_data_to_url.py | 74 ++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 mirrors/migrations/0019_move_country_data_to_url.py (limited to 'mirrors') diff --git a/mirrors/migrations/0019_move_country_data_to_url.py b/mirrors/migrations/0019_move_country_data_to_url.py new file mode 100644 index 00000000..81b7bb3e --- /dev/null +++ b/mirrors/migrations/0019_move_country_data_to_url.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import DataMigration +from django.db import models + +class Migration(DataMigration): + + def forwards(self, orm): + for url in orm.MirrorUrl.objects.select_related('mirror').all(): + # set the country field on the URL if we have one, + # and it isn't already set to anything. + if url.country or not url.mirror.country: + continue + orm.MirrorUrl.objects.filter(pk=url.pk).update( + country=url.mirror.country) + + def backwards(self, orm): + pass + + models = { + 'mirrors.mirror': { + 'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + 'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) + }, + 'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) + }, + 'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] + symmetrical = True -- cgit v1.2.3-2-g168b From 6f0ae6746baea657ee6d7c21ac0813a04f825443 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 14 Jan 2013 01:00:11 -0600 Subject: Drop country column from mirror table We now always look for this information at the URL level, not the mirror level. This simplifies quite a bit of code in and around the mirror views. Signed-off-by: Dan McGee --- mirrors/admin.py | 4 +- .../0020_auto__del_field_mirror_country.py | 70 ++++++++++++++++++++++ mirrors/models.py | 7 +-- mirrors/utils.py | 7 +-- mirrors/views.py | 15 ++--- 5 files changed, 81 insertions(+), 22 deletions(-) create mode 100644 mirrors/migrations/0020_auto__del_field_mirror_country.py (limited to 'mirrors') diff --git a/mirrors/admin.py b/mirrors/admin.py index 65fff368..eaa38391 100644 --- a/mirrors/admin.py +++ b/mirrors/admin.py @@ -62,9 +62,9 @@ class MirrorAdminForm(forms.ModelForm): class MirrorAdmin(admin.ModelAdmin): form = MirrorAdminForm - list_display = ('name', 'tier', 'country', 'active', 'public', + list_display = ('name', 'tier', 'active', 'public', 'isos', 'admin_email', 'alternate_email') - list_filter = ('tier', 'active', 'public', 'country') + list_filter = ('tier', 'active', 'public') search_fields = ('name', 'admin_email', 'alternate_email') inlines = [ MirrorUrlInlineAdmin, diff --git a/mirrors/migrations/0020_auto__del_field_mirror_country.py b/mirrors/migrations/0020_auto__del_field_mirror_country.py new file mode 100644 index 00000000..c2220a50 --- /dev/null +++ b/mirrors/migrations/0020_auto__del_field_mirror_country.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.delete_column('mirrors_mirror', 'country') + + + def backwards(self, orm): + db.add_column('mirrors_mirror', 'country', + self.gf('django_countries.fields.CountryField')(blank=True, default='', max_length=2, db_index=True), + keep_default=False) + + + models = { + 'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + 'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) + }, + 'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) + }, + 'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index 0179d5bf..ca421d13 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -17,7 +17,6 @@ class Mirror(models.Model): name = models.CharField(max_length=255, unique=True) tier = models.SmallIntegerField(default=2, choices=TIER_CHOICES) upstream = models.ForeignKey('self', null=True, on_delete=models.SET_NULL) - country = CountryField(blank=True, db_index=True) admin_email = models.EmailField(max_length=255, blank=True) alternate_email = models.EmailField(max_length=255, blank=True) public = models.BooleanField(default=True) @@ -28,7 +27,7 @@ class Mirror(models.Model): notes = models.TextField(blank=True) class Meta: - ordering = ('country', 'name') + ordering = ('name',) def __unicode__(self): return self.name @@ -75,10 +74,6 @@ class MirrorUrl(models.Model): def hostname(self): return urlparse(self.url).hostname - @property - def real_country(self): - return self.country or self.mirror.country - def clean(self): try: # Auto-map the protocol field by looking at the URL diff --git a/mirrors/utils.py b/mirrors/utils.py index 1d560021..3ab176b3 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -115,7 +115,7 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None): is_success=False, check_time__gte=cutoff_time, url__mirror__active=True, url__mirror__public=True).values( 'url__url', 'url__country', 'url__protocol__protocol', - 'url__mirror__country', 'url__mirror__tier', 'error').annotate( + 'url__mirror__tier', 'error').annotate( error_count=Count('error'), last_occurred=Max('check_time') ).order_by('-last_occurred', '-error_count') @@ -124,8 +124,7 @@ def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_ids=None): errors = list(errors) for err in errors: - ctry_code = err['url__country'] or err['url__mirror__country'] - err['country'] = Country(ctry_code) + err['country'] = Country(err['url__country']) return errors @@ -152,7 +151,7 @@ def get_mirror_url_for_download(cutoff=DEFAULT_CUTOFF): mirror_urls = MirrorUrl.objects.filter( mirror__public=True, mirror__active=True, protocol__default=True) # look first for a country-agnostic URL, then fall back to any HTTP URL - filtered_urls = mirror_urls.filter(mirror__country='')[:1] + filtered_urls = mirror_urls.filter(country='')[:1] if not filtered_urls: filtered_urls = mirror_urls[:1] if not filtered_urls: diff --git a/mirrors/views.py b/mirrors/views.py index d3867802..545e3557 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -42,9 +42,6 @@ class MirrorlistForm(forms.Form): def get_countries(self): country_codes = set() - country_codes.update(Mirror.objects.filter(active=True).exclude( - country='').values_list( - 'country', flat=True).order_by().distinct()) country_codes.update(MirrorUrl.objects.filter( mirror__active=True).exclude(country='').values_list( 'country', flat=True).order_by().distinct()) @@ -81,7 +78,7 @@ def generate_mirrorlist(request): def default_protocol_filter(original_urls): - key_func = attrgetter('real_country') + key_func = attrgetter('country') sorted_urls = sorted(original_urls, key=key_func) urls = [] for _, group in groupby(sorted_urls, key=key_func): @@ -119,8 +116,7 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, protocol__in=protocols, mirror__public=True, mirror__active=True) if countries and 'all' not in countries: - qset = qset.filter(Q(country__in=countries) | - Q(mirror__country__in=countries)) + qset = qset.filter(country__in=countries) ip_version = Q() if ipv4_supported: @@ -135,7 +131,7 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, urls = qset if not use_status: - sort_key = attrgetter('real_country.name', 'mirror.name', 'url') + sort_key = attrgetter('country.name', 'mirror.name', 'url') urls = sorted(urls, key=sort_key) template = 'mirrors/mirrorlist.txt' else: @@ -158,7 +154,7 @@ def find_mirrors_simple(request, protocol): def mirrors(request): - mirror_list = Mirror.objects.select_related().order_by('tier', 'country') + mirror_list = Mirror.objects.select_related().order_by('tier', 'name') protos = MirrorUrl.objects.values_list( 'mirror_id', 'protocol__protocol').order_by( 'mirror__id', 'protocol__protocol').distinct() @@ -254,8 +250,7 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder): return list(obj) if isinstance(obj, MirrorUrl): data = {attr: getattr(obj, attr) for attr in self.url_attributes} - # get any override on the country attribute first - country = obj.real_country + country = obj.country data['country'] = unicode(country.name) data['country_code'] = country.code return data -- cgit v1.2.3-2-g168b From ff6db38f1dc6ed1eb53454a7e16615ec1ad76d7a Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 14 Jan 2013 01:27:34 -0600 Subject: Ensure URLs without check data work on mirror details page Less noticeable in production as the templates don't show '@@@INVALID@@@' there, but we were trying to access attributes that don't actually exist on certain mirror objects. Signed-off-by: Dan McGee --- mirrors/views.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'mirrors') diff --git a/mirrors/views.py b/mirrors/views.py index 545e3557..30df5472 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -176,13 +176,17 @@ def mirror_details(request, name): raise Http404 status_info = get_mirror_statuses(mirror_ids=[mirror.id]) - checked_urls = [url for url in status_info['urls'] \ - if url.mirror_id == mirror.id] - all_urls = mirror.urls.select_related('protocol') - # get each item from checked_urls and supplement with anything in all_urls - # if it wasn't there - all_urls = set(checked_urls).union(all_urls) - all_urls = sorted(all_urls, key=attrgetter('url')) + checked_urls = {url for url in status_info['urls'] \ + if url.mirror_id == mirror.id} + all_urls = set(mirror.urls.select_related('protocol')) + # Add dummy data for URLs that we haven't checked recently + other_urls = all_urls.difference(checked_urls) + print other_urls + for url in other_urls: + for attr in ('last_sync', 'completion_pct', 'delay', 'duration_avg', + 'duration_stddev', 'score'): + setattr(url, attr, None) + all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url')) return render(request, 'mirrors/mirror_details.html', {'mirror': mirror, 'urls': all_urls}) -- cgit v1.2.3-2-g168b From 0f6a0a1cd0011c8ad137a4b27d0b39a7e1129fb7 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 14 Jan 2013 08:54:33 -0600 Subject: Support mirror status JSON by tier Just as we do for the normal status HTML view. Signed-off-by: Dan McGee --- mirrors/urls.py | 1 + mirrors/views.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/urls.py b/mirrors/urls.py index 857e99e2..4e929410 100644 --- a/mirrors/urls.py +++ b/mirrors/urls.py @@ -5,6 +5,7 @@ urlpatterns = patterns('mirrors.views', (r'^status/$', 'status', {}, 'mirror-status'), (r'^status/json/$', 'status_json', {}, 'mirror-status-json'), (r'^status/tier/(?P\d+)/$', 'status', {}, 'mirror-status-tier'), + (r'^status/tier/(?P\d+)/json/$', 'status_json', {}, 'mirror-status-tier-json'), (r'^(?P[\.\-\w]+)/$', 'mirror_details'), (r'^(?P[\.\-\w]+)/json/$', 'mirror_details_json'), ) diff --git a/mirrors/views.py b/mirrors/views.py index 30df5472..c0ed6670 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -279,9 +279,15 @@ class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) -def status_json(request): +def status_json(request, tier=None): + if tier is not None: + tier = int(tier) + if tier not in [t[0] for t in Mirror.TIER_CHOICES]: + raise Http404 status_info = get_mirror_statuses() data = status_info.copy() + if tier is not None: + data['urls'] = [url for url in data['urls'] if url.mirror.tier == tier] data['version'] = 3 to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder) response = HttpResponse(to_json, content_type='application/json') -- cgit v1.2.3-2-g168b From 1f9aef78f39c90191eddf2233c278086a15052de Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 16 Jan 2013 00:18:26 -0600 Subject: Remove debugging print statement Signed-off-by: Dan McGee --- mirrors/views.py | 1 - 1 file changed, 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/views.py b/mirrors/views.py index c0ed6670..56397633 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -181,7 +181,6 @@ def mirror_details(request, name): all_urls = set(mirror.urls.select_related('protocol')) # Add dummy data for URLs that we haven't checked recently other_urls = all_urls.difference(checked_urls) - print other_urls for url in other_urls: for attr in ('last_sync', 'completion_pct', 'delay', 'duration_avg', 'duration_stddev', 'score'): -- cgit v1.2.3-2-g168b From dd8e94f69783365160bcbfda61a9bebea5a71324 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 20 Jan 2013 14:53:28 -0600 Subject: Lengthen the mirror rsync IP address field Make it long enough to support a full-form IPv6 address with a subnet. Signed-off-by: Dan McGee --- .../0021_auto__chg_field_mirrorrsync_ip.py | 66 ++++++++++++++++++++++ mirrors/models.py | 5 +- 2 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 mirrors/migrations/0021_auto__chg_field_mirrorrsync_ip.py (limited to 'mirrors') diff --git a/mirrors/migrations/0021_auto__chg_field_mirrorrsync_ip.py b/mirrors/migrations/0021_auto__chg_field_mirrorrsync_ip.py new file mode 100644 index 00000000..bbf14bb0 --- /dev/null +++ b/mirrors/migrations/0021_auto__chg_field_mirrorrsync_ip.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.alter_column(u'mirrors_mirrorrsync', 'ip', self.gf('django.db.models.fields.CharField')(max_length=44)) + + def backwards(self, orm): + db.alter_column(u'mirrors_mirrorrsync', 'ip', self.gf('django.db.models.fields.CharField')(max_length=24)) + + models = { + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index ca421d13..ec4a044d 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -98,11 +98,12 @@ class MirrorUrl(models.Model): class MirrorRsync(models.Model): - ip = models.CharField("IP", max_length=24) + # max length is 40 chars for full-form IPv6 addr + subnet + ip = models.CharField("IP", max_length=44) mirror = models.ForeignKey(Mirror, related_name="rsync_ips") def __unicode__(self): - return "%s" % (self.ip) + return self.ip class Meta: verbose_name = 'mirror rsync IP' -- cgit v1.2.3-2-g168b From 5566d43a7734f6bb2f48d5d511351da12ddc5cc1 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 9 Feb 2013 16:43:40 -0600 Subject: Use 'update_fields' model.save() kwarg This was added in Django 1.5 and allows saving only a subset of a model's fields. It makes sense in a few cases to utilize it. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorresolv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorresolv.py b/mirrors/management/commands/mirrorresolv.py index 0370f8ed..a6c2523e 100644 --- a/mirrors/management/commands/mirrorresolv.py +++ b/mirrors/management/commands/mirrorresolv.py @@ -53,7 +53,7 @@ def resolve_mirrors(): newvals = (mirrorurl.has_ipv4, mirrorurl.has_ipv6) if newvals != oldvals: logger.debug("values changed for %s", mirrorurl) - mirrorurl.save(force_update=True) + mirrorurl.save(update_fields=('has_ipv4', 'has_ipv6')) except socket.error, e: logger.warn("error resolving %s: %s", mirrorurl.hostname, e) -- cgit v1.2.3-2-g168b From 10af269ed21b00cb3da9b380f72016fefae52a6d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 4 Mar 2013 18:11:28 -0600 Subject: Add HTTPS mirror protocol fixture For now, it is not included in the default selection, but we have a few existing mirrors that do support it. Signed-off-by: Dan McGee --- mirrors/fixtures/mirrorprotocols.json | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'mirrors') diff --git a/mirrors/fixtures/mirrorprotocols.json b/mirrors/fixtures/mirrorprotocols.json index 72ed1a7f..8822ef8e 100644 --- a/mirrors/fixtures/mirrorprotocols.json +++ b/mirrors/fixtures/mirrorprotocols.json @@ -7,7 +7,7 @@ "default": true, "protocol": "http" } - }, + }, { "pk": 2, "model": "mirrors.mirrorprotocol", @@ -16,7 +16,7 @@ "default": false, "protocol": "ftp" } - }, + }, { "pk": 3, "model": "mirrors.mirrorprotocol", @@ -25,5 +25,14 @@ "default": false, "protocol": "rsync" } + }, + { + "pk": 5, + "model": "mirrors.mirrorprotocol", + "fields": { + "is_download": true, + "default": false, + "protocol": "https" + } } ] -- cgit v1.2.3-2-g168b From d158ce71e4ec489ee3ec1a73c41c9b9dc8d34a23 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 6 Mar 2013 19:40:24 -0600 Subject: Add mirror check locations model Signed-off-by: Dan McGee --- mirrors/migrations/0022_auto__add_checklocation.py | 83 ++++++++++++++++++++++ mirrors/models.py | 23 +++++- 2 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 mirrors/migrations/0022_auto__add_checklocation.py (limited to 'mirrors') diff --git a/mirrors/migrations/0022_auto__add_checklocation.py b/mirrors/migrations/0022_auto__add_checklocation.py new file mode 100644 index 00000000..896b2dab --- /dev/null +++ b/mirrors/migrations/0022_auto__add_checklocation.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.create_table(u'mirrors_checklocation', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('hostname', self.gf('django.db.models.fields.CharField')(max_length=255)), + ('source_ip', self.gf('django.db.models.fields.GenericIPAddressField')(unique=True, max_length=39)), + ('country', self.gf('django_countries.fields.CountryField')(max_length=2)), + ('created', self.gf('django.db.models.fields.DateTimeField')()), + )) + db.send_create_signal(u'mirrors', ['CheckLocation']) + + + def backwards(self, orm): + db.delete_table(u'mirrors_checklocation') + + + models = { + u'mirrors.checklocation': { + 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'}, + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'}) + }, + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index ec4a044d..c7a0a93f 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -1,10 +1,13 @@ import socket from urlparse import urlparse -from django.db import models from django.core.exceptions import ValidationError +from django.db import models +from django.db.models.signals import pre_save from django_countries import CountryField +from main.utils import set_created_field + class Mirror(models.Model): TIER_CHOICES = ( @@ -109,6 +112,20 @@ class MirrorRsync(models.Model): verbose_name = 'mirror rsync IP' +class CheckLocation(models.Model): + hostname = models.CharField(max_length=255) + source_ip = models.GenericIPAddressField(verbose_name='source IP', + unpack_ipv4=True, unique=True) + country = CountryField() + created = models.DateTimeField(editable=False) + + class Meta: + ordering = ('hostname', 'source_ip') + + def __unicode__(self): + return self.hostname + + class MirrorLog(models.Model): url = models.ForeignKey(MirrorUrl, related_name="logs") check_time = models.DateTimeField(db_index=True) @@ -124,4 +141,8 @@ class MirrorLog(models.Model): verbose_name = 'mirror check log' get_latest_by = 'check_time' + +pre_save.connect(set_created_field, sender=CheckLocation, + dispatch_uid="mirrors.models") + # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From 1dbf311774f7894cac870517558d8baee8681f0d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 6 Mar 2013 19:45:52 -0600 Subject: Add 'created' field to more mirror models We have been better about doing this to most of our models, but the ones here didn't have a created field. Add it where appropriate and set a reasonably old default value. Signed-off-by: Dan McGee --- ...created__add_field_mirrorrsync_created__add_.py | 97 ++++++++++++++++++++++ mirrors/models.py | 9 +- 2 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 mirrors/migrations/0023_auto__add_field_mirrorurl_created__add_field_mirrorrsync_created__add_.py (limited to 'mirrors') diff --git a/mirrors/migrations/0023_auto__add_field_mirrorurl_created__add_field_mirrorrsync_created__add_.py b/mirrors/migrations/0023_auto__add_field_mirrorurl_created__add_field_mirrorrsync_created__add_.py new file mode 100644 index 00000000..1a1f48a0 --- /dev/null +++ b/mirrors/migrations/0023_auto__add_field_mirrorurl_created__add_field_mirrorrsync_created__add_.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models +from pytz import utc + + +class Migration(SchemaMigration): + + def forwards(self, orm): + default = datetime.datetime(2000, 1, 1, 0, 0).replace(tzinfo=utc) + db.add_column(u'mirrors_mirrorurl', 'created', + self.gf('django.db.models.fields.DateTimeField')(default=default), + keep_default=False) + db.add_column(u'mirrors_mirrorrsync', 'created', + self.gf('django.db.models.fields.DateTimeField')(default=default), + keep_default=False) + db.add_column(u'mirrors_mirrorprotocol', 'created', + self.gf('django.db.models.fields.DateTimeField')(default=default), + keep_default=False) + db.add_column(u'mirrors_mirror', 'created', + self.gf('django.db.models.fields.DateTimeField')(default=default), + keep_default=False) + + + def backwards(self, orm): + db.delete_column(u'mirrors_mirrorurl', 'created') + db.delete_column(u'mirrors_mirrorrsync', 'created') + db.delete_column(u'mirrors_mirrorprotocol', 'created') + db.delete_column(u'mirrors_mirror', 'created') + + + models = { + u'mirrors.checklocation': { + 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'}, + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'}) + }, + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index c7a0a93f..c205fef2 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -28,6 +28,7 @@ class Mirror(models.Model): rsync_user = models.CharField(max_length=50, blank=True, default='') rsync_password = models.CharField(max_length=50, blank=True, default='') notes = models.TextField(blank=True) + created = models.DateTimeField(editable=False) class Meta: ordering = ('name',) @@ -48,6 +49,7 @@ class MirrorProtocol(models.Model): help_text="Is protocol useful for end-users, e.g. FTP/HTTP") default = models.BooleanField(default=True, help_text="Included by default when building mirror list?") + created = models.DateTimeField(editable=False) def __unicode__(self): return self.protocol @@ -66,6 +68,7 @@ class MirrorUrl(models.Model): editable=False) has_ipv6 = models.BooleanField("IPv6 capable", default=False, editable=False) + created = models.DateTimeField(editable=False) def address_families(self): hostname = urlparse(self.url).hostname @@ -104,6 +107,7 @@ class MirrorRsync(models.Model): # max length is 40 chars for full-form IPv6 addr + subnet ip = models.CharField("IP", max_length=44) mirror = models.ForeignKey(Mirror, related_name="rsync_ips") + created = models.DateTimeField(editable=False) def __unicode__(self): return self.ip @@ -142,7 +146,8 @@ class MirrorLog(models.Model): get_latest_by = 'check_time' -pre_save.connect(set_created_field, sender=CheckLocation, - dispatch_uid="mirrors.models") +for model in (Mirror, MirrorProtocol, MirrorUrl, MirrorRsync, CheckLocation): + pre_save.connect(set_created_field, sender=model, + dispatch_uid="mirrors.models") # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From 71259ab4c27ca6f00e09e813c7d9c6e8e24d59b4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 6 Mar 2013 19:53:11 -0600 Subject: Add mirror CheckLocationAdmin Signed-off-by: Dan McGee --- mirrors/admin.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/admin.py b/mirrors/admin.py index eaa38391..d6ea3950 100644 --- a/mirrors/admin.py +++ b/mirrors/admin.py @@ -4,7 +4,9 @@ from urlparse import urlparse, urlunsplit from django import forms from django.contrib import admin -from .models import Mirror, MirrorProtocol, MirrorUrl, MirrorRsync +from .models import (Mirror, MirrorProtocol, MirrorUrl, MirrorRsync, + CheckLocation) + class MirrorUrlForm(forms.ModelForm): class Meta: @@ -26,12 +28,14 @@ class MirrorUrlForm(forms.ModelForm): url = urlunsplit((url_parts.scheme, url_parts.netloc, path, '', '')) return url + class MirrorUrlInlineAdmin(admin.TabularInline): model = MirrorUrl form = MirrorUrlForm readonly_fields = ('protocol', 'has_ipv4', 'has_ipv6') extra = 3 + # ripped off from django.forms.fields, adding netmask ability IPV4NM_RE = re.compile(r'^(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}(/(\d|[1-2]\d|3[0-2])){0,1}$') @@ -43,16 +47,19 @@ class IPAddressNetmaskField(forms.fields.RegexField): def __init__(self, *args, **kwargs): super(IPAddressNetmaskField, self).__init__(IPV4NM_RE, *args, **kwargs) + class MirrorRsyncForm(forms.ModelForm): class Meta: model = MirrorRsync ip = IPAddressNetmaskField(label='IP') + class MirrorRsyncInlineAdmin(admin.TabularInline): model = MirrorRsync form = MirrorRsyncForm extra = 2 + class MirrorAdminForm(forms.ModelForm): class Meta: model = Mirror @@ -60,6 +67,7 @@ class MirrorAdminForm(forms.ModelForm): queryset=Mirror.objects.filter(tier__gte=0, tier__lte=1), required=False) + class MirrorAdmin(admin.ModelAdmin): form = MirrorAdminForm list_display = ('name', 'tier', 'active', 'public', @@ -71,11 +79,19 @@ class MirrorAdmin(admin.ModelAdmin): MirrorRsyncInlineAdmin, ] + class MirrorProtocolAdmin(admin.ModelAdmin): list_display = ('protocol', 'is_download', 'default') list_filter = ('is_download', 'default') + +class CheckLocationAdmin(admin.ModelAdmin): + list_display = ('hostname', 'source_ip', 'country', 'created') + search_fields = ('hostname', 'source_ip') + + admin.site.register(Mirror, MirrorAdmin) admin.site.register(MirrorProtocol, MirrorProtocolAdmin) +admin.site.register(CheckLocation, CheckLocationAdmin) # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From 3e0209f5e8ee197034b6c1f705af515d8154801b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 6 Mar 2013 19:56:48 -0600 Subject: Revert "mirrorcheck: Don't use bulk_create on sqlite3" This reverts commit 3c4ceb16. We don't need this anymore as bulk_create gets automatic batching now on sqlite3 so it is safe to use. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index e09ea680..2116ab29 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -31,7 +31,6 @@ from django.core.management.base import NoArgsCommand from django.db import transaction from django.utils.timezone import now -from main.utils import database_vendor from mirrors.models import MirrorUrl, MirrorLog logging.basicConfig( @@ -208,11 +207,7 @@ class MirrorCheckPool(object): logger.debug("joining on all threads") self.tasks.join() logger.debug("processing %d log entries", len(self.logs)) - if database_vendor(MirrorLog, mode='write') == 'sqlite': - for log in self.logs: - log.save(force_insert=True) - else: - MirrorLog.objects.bulk_create(self.logs) + MirrorLog.objects.bulk_create(self.logs) logger.debug("log entries saved") # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From d9dbe4fb1ebbf4c5d26b151509acc4ce30654b8d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 6 Mar 2013 20:43:09 -0600 Subject: Add family property to mirror check location Signed-off-by: Dan McGee --- mirrors/models.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'mirrors') diff --git a/mirrors/models.py b/mirrors/models.py index c205fef2..07ac1e6e 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -129,6 +129,13 @@ class CheckLocation(models.Model): def __unicode__(self): return self.hostname + @property + def family(self): + info = socket.getaddrinfo(self.source_ip, None, 0, 0, 0, + socket.AI_NUMERICHOST) + families = [x[0] for x in info] + return families[0] + class MirrorLog(models.Model): url = models.ForeignKey(MirrorUrl, related_name="logs") -- cgit v1.2.3-2-g168b From 9917ee3482d6fa91f779af9ad2d02097775211a4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 6 Mar 2013 20:49:38 -0600 Subject: Add location ID to mirror logs Signed-off-by: Dan McGee --- .../0024_auto__add_field_mirrorlog_location.py | 83 ++++++++++++++++++++++ mirrors/models.py | 1 + 2 files changed, 84 insertions(+) create mode 100644 mirrors/migrations/0024_auto__add_field_mirrorlog_location.py (limited to 'mirrors') diff --git a/mirrors/migrations/0024_auto__add_field_mirrorlog_location.py b/mirrors/migrations/0024_auto__add_field_mirrorlog_location.py new file mode 100644 index 00000000..acf8df17 --- /dev/null +++ b/mirrors/migrations/0024_auto__add_field_mirrorlog_location.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + db.add_column(u'mirrors_mirrorlog', 'location', + self.gf('django.db.models.fields.related.ForeignKey')(related_name='logs', null=True, to=orm['mirrors.CheckLocation']), + keep_default=False) + + + def backwards(self, orm): + db.delete_column(u'mirrors_mirrorlog', 'location_id') + + + models = { + u'mirrors.checklocation': { + 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'}, + 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'}) + }, + u'mirrors.mirror': { + 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'}) + }, + u'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'location': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'null': 'True', 'to': u"orm['mirrors.CheckLocation']"}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"}) + }, + u'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + u'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'created': ('django.db.models.fields.DateTimeField', [], {}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '44'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"}) + }, + u'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index 07ac1e6e..e41f6b22 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -139,6 +139,7 @@ class CheckLocation(models.Model): class MirrorLog(models.Model): url = models.ForeignKey(MirrorUrl, related_name="logs") + location = models.ForeignKey(CheckLocation, related_name="logs", null=True) check_time = models.DateTimeField(db_index=True) last_sync = models.DateTimeField(null=True) duration = models.FloatField(null=True) -- cgit v1.2.3-2-g168b From 7c8b09b95ce5db9ddf7e895c2722bd202f5c4f54 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 6 Mar 2013 20:58:09 -0600 Subject: Teach mirrorcheck management command about check locations This adds the -l/--location argument to the command in order to pass in a check location that we are currently running from. This locks the IP address family to the one derived from the address on that location, and stores any check results tagged with a location ID. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 63 ++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 17 deletions(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 2116ab29..f133c785 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -31,7 +31,8 @@ from django.core.management.base import NoArgsCommand from django.db import transaction from django.utils.timezone import now -from mirrors.models import MirrorUrl, MirrorLog +from mirrors.models import MirrorUrl, MirrorLog, CheckLocation + logging.basicConfig( level=logging.WARNING, @@ -40,17 +41,17 @@ logging.basicConfig( stream=sys.stderr) logger = logging.getLogger() - class Command(NoArgsCommand): option_list = NoArgsCommand.option_list + ( - make_option('-t', '--timeout', dest='timeout', default='10', + make_option('-t', '--timeout', dest='timeout', type='float', default=10.0, help='Timeout value for connecting to URL'), + make_option('-l', '--location', dest='location', type='int', + help='ID of CheckLocation object to use for this run'), ) help = "Runs a check on all known mirror URLs to determine their up-to-date status." def handle_noargs(self, **options): v = int(options.get('verbosity', 0)) - timeout = int(options.get('timeout', 10)) if v == 0: logger.level = logging.ERROR elif v == 1: @@ -58,14 +59,35 @@ class Command(NoArgsCommand): elif v == 2: logger.level = logging.DEBUG + timeout = options.get('timeout') + urls = MirrorUrl.objects.select_related('protocol').filter( mirror__active=True, mirror__public=True) - pool = MirrorCheckPool(urls, timeout) + location = options.get('location', None) + if location: + location = CheckLocation.objects.get(id=location) + family = location.family + monkeypatch_getaddrinfo(family) + if family == socket.AF_INET6: + urls = urls.filter(has_ipv6=True) + elif family == socket.AF_INET: + urls = urls.filter(has_ipv4=True) + + pool = MirrorCheckPool(urls, location, timeout) pool.run() return 0 +def monkeypatch_getaddrinfo(force_family=socket.AF_INET): + '''Force the Python socket module to connect over the designated family; + e.g. socket.AF_INET or socket.AF_INET6.''' + orig = socket.getaddrinfo + def wrapper(host, port, family=0, socktype=0, proto=0, flags=0): + return orig(host, port, force_family, socktype, proto, flags) + socket.getaddrinfo = wrapper + + def parse_lastsync(log, data): '''lastsync file should be an epoch value created by us.''' try: @@ -80,10 +102,10 @@ def parse_lastsync(log, data): log.is_success = False -def check_mirror_url(mirror_url, timeout): +def check_mirror_url(mirror_url, location, timeout): url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) - log = MirrorLog(url=mirror_url, check_time=now()) + log = MirrorLog(url=mirror_url, check_time=now(), location=location) headers = {'User-Agent': 'archweb/1.0'} req = urllib2.Request(url, None, headers) try: @@ -133,17 +155,24 @@ def check_mirror_url(mirror_url, timeout): return log -def check_rsync_url(mirror_url, timeout): +def check_rsync_url(mirror_url, location, timeout): url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) - log = MirrorLog(url=mirror_url, check_time=now()) + log = MirrorLog(url=mirror_url, check_time=now(), location=location) tempdir = tempfile.mkdtemp() + ipopt = '' + if location: + if location.family == socket.AF_INET6: + ipopt = '--ipv6' + elif location.family == socket.AF_INET: + ipopt = '--ipv4' lastsync_path = os.path.join(tempdir, 'lastsync') rsync_cmd = ["rsync", "--quiet", "--contimeout=%d" % timeout, - "--timeout=%d" % timeout, url, lastsync_path] + "--timeout=%d" % timeout, ipopt, url, lastsync_path] try: with open(os.devnull, 'w') as devnull: + logger.debug("rsync cmd: %s", ' '.join(rsync_cmd)) proc = subprocess.Popen(rsync_cmd, stdout=devnull, stderr=subprocess.PIPE) start = time.time() @@ -170,15 +199,15 @@ def check_rsync_url(mirror_url, timeout): return log -def mirror_url_worker(work, output, timeout): +def mirror_url_worker(work, output, location, timeout): while True: try: url = work.get(block=False) try: if url.protocol.protocol == 'rsync': - log = check_rsync_url(url, timeout) + log = check_rsync_url(url, location, timeout) else: - log = check_mirror_url(url, timeout) + log = check_mirror_url(url, location, timeout) output.append(log) finally: work.task_done() @@ -187,15 +216,15 @@ def mirror_url_worker(work, output, timeout): class MirrorCheckPool(object): - def __init__(self, urls, timeout=10, num_threads=10): + def __init__(self, urls, location, timeout=10, num_threads=10): self.tasks = Queue() self.logs = deque() - for i in list(urls): - self.tasks.put(i) + for url in list(urls): + self.tasks.put(url) self.threads = [] for i in range(num_threads): thread = Thread(target=mirror_url_worker, - args=(self.tasks, self.logs, timeout)) + args=(self.tasks, self.logs, location, timeout)) thread.daemon = True self.threads.append(thread) -- cgit v1.2.3-2-g168b From ace95f6e53f41409568d4e4f1cf4c2a69d931e2c Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 6 Mar 2013 21:38:58 -0600 Subject: Don't add blank options to rsync command line Rsync doesn't like this so much: Unexpected remote arg: rsync://mirror.example.com/archlinux/lastsync rsync error: syntax or usage error (code 1) at main.c(1214) [sender=3.0.9] Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index f133c785..1315a013 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -169,7 +169,11 @@ def check_rsync_url(mirror_url, location, timeout): ipopt = '--ipv4' lastsync_path = os.path.join(tempdir, 'lastsync') rsync_cmd = ["rsync", "--quiet", "--contimeout=%d" % timeout, - "--timeout=%d" % timeout, ipopt, url, lastsync_path] + "--timeout=%d" % timeout] + if ipopt: + rsync_cmd.append(ipopt) + rsync_cmd.append(url) + rsync_cmd.append(lastsync_path) try: with open(os.devnull, 'w') as devnull: logger.debug("rsync cmd: %s", ' '.join(rsync_cmd)) -- cgit v1.2.3-2-g168b From 46d21e03e81e4cacc849d798052b3ffd525d638a Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 12 Mar 2013 20:18:51 -0500 Subject: Don't check FTP + IPv6 combination Very few, if any, FTP servers support connections over IPv6. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 1315a013..93b53d6b 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -41,6 +41,7 @@ logging.basicConfig( stream=sys.stderr) logger = logging.getLogger() + class Command(NoArgsCommand): option_list = NoArgsCommand.option_list + ( make_option('-t', '--timeout', dest='timeout', type='float', default=10.0, @@ -83,8 +84,10 @@ def monkeypatch_getaddrinfo(force_family=socket.AF_INET): '''Force the Python socket module to connect over the designated family; e.g. socket.AF_INET or socket.AF_INET6.''' orig = socket.getaddrinfo + def wrapper(host, port, family=0, socktype=0, proto=0, flags=0): return orig(host, port, force_family, socktype, proto, flags) + socket.getaddrinfo = wrapper @@ -103,6 +106,12 @@ def parse_lastsync(log, data): def check_mirror_url(mirror_url, location, timeout): + if location: + if location.family == socket.AF_INET6: + ipopt = '--ipv6' + elif location.family == socket.AF_INET: + ipopt = '--ipv4' + url = mirror_url.url + 'lastsync' logger.info("checking URL %s", url) log = MirrorLog(url=mirror_url, check_time=now(), location=location) @@ -210,9 +219,14 @@ def mirror_url_worker(work, output, location, timeout): try: if url.protocol.protocol == 'rsync': log = check_rsync_url(url, location, timeout) + if (url.protocol.protocol == 'ftp' and location and + location.family == socket.AF_INET6): + # IPv6 + FTP don't work; skip checking completely + log = None else: log = check_mirror_url(url, location, timeout) - output.append(log) + if log: + output.append(log) finally: work.task_done() except Empty: -- cgit v1.2.3-2-g168b From b8ee7b1ee281b45b245fb454228b8ad847c56200 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 13 Mar 2013 13:36:14 -0500 Subject: mirrorcheck: s/if/elif/ when determining what check function to run This was a silly thinko here; it caused the logs to fill up with a bunch of 'unknown url type: rsync' errors. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 93b53d6b..d6de8f22 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -219,7 +219,7 @@ def mirror_url_worker(work, output, location, timeout): try: if url.protocol.protocol == 'rsync': log = check_rsync_url(url, location, timeout) - if (url.protocol.protocol == 'ftp' and location and + elif (url.protocol.protocol == 'ftp' and location and location.family == socket.AF_INET6): # IPv6 + FTP don't work; skip checking completely log = None -- cgit v1.2.3-2-g168b