From d2d0895f13835569ff25a3161ddb94cd655dfd4f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 2 May 2012 12:23:21 -0500 Subject: Allow mirrorlist generator pattern to match any protocol Add a helper method that checks if we know about the protocol; if so, we can spit out a URL for it. This allows (if you are insane) generation of an rsync mirrorlist, for instance. Signed-off-by: Dan McGee --- mirrors/views.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index c52656f7..6f37ace1 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -82,12 +82,15 @@ def generate_mirrorlist(request): def find_mirrors(request, countries=None, protocols=None, use_status=False, ipv4_supported=True, ipv6_supported=True): if not protocols: - protocols = MirrorProtocol.objects.filter( - is_download=True).values_list('protocol', flat=True) + protocols = MirrorProtocol.objects.filter(is_download=True) + elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol: + # we already have a queryset, no need to query again + pass + else: + protocols = MirrorProtocol.objects.filter(protocol__in=protocols) qset = MirrorUrl.objects.select_related().filter( - protocol__protocol__in=protocols, - mirror__public=True, mirror__active=True, - ) + protocol__in=protocols, + mirror__public=True, mirror__active=True) if countries and 'all' not in countries: qset = qset.filter(Q(country__in=countries) | Q(mirror__country__in=countries)) @@ -124,6 +127,11 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, mimetype='text/plain') +def find_mirrors_simple(request, protocol): + proto = get_object_or_404(MirrorProtocol, protocol=protocol) + return find_mirrors(request, protocols=[proto]) + + def mirrors(request): mirror_list = Mirror.objects.select_related().order_by('tier', 'country') if not request.user.is_authenticated(): -- cgit v1.2.3-2-g168b From 12bf4c1b1e7df2d934b9dfde8629137dedeea99f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 18 Feb 2012 19:17:00 -0600 Subject: Add a smart protocol filter to mirrorlist generator This will only list FTP mirrors for a given country if there are no HTTP mirrors available, since FTP must die. Signed-off-by: Dan McGee --- mirrors/views.py | 55 ++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 15 deletions(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index 6f37ace1..eac78ff2 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -1,4 +1,5 @@ from datetime import timedelta +from itertools import groupby from operator import attrgetter, itemgetter from django import forms @@ -79,8 +80,34 @@ def generate_mirrorlist(request): {'mirrorlist_form': form}) +def default_protocol_filter(original_urls): + key_func = attrgetter('real_country') + sorted_urls = sorted(original_urls, key=key_func) + urls = [] + for _, group in groupby(sorted_urls, key=key_func): + cntry_urls = list(group) + if any(url.protocol.default for url in cntry_urls): + cntry_urls = [url for url in cntry_urls if url.protocol.default] + urls.extend(cntry_urls) + return urls + + +def status_filter(original_urls): + status_info = get_mirror_statuses() + scores = dict((u.id, u.score) for u in status_info['urls']) + urls = [] + for u in original_urls: + u.score = scores.get(u.id, None) + # also include mirrors that don't have an up to date score + # (as opposed to those that have been set with no score) + if (u.id not in scores) or (u.score and u.score < 100.0): + urls.append(u) + # if a url doesn't have a score, treat it as the highest possible + return sorted(urls, key=lambda x: x.score or 100.0) + + def find_mirrors(request, countries=None, protocols=None, use_status=False, - ipv4_supported=True, ipv6_supported=True): + ipv4_supported=True, ipv6_supported=True, smart_protocol=False): if not protocols: protocols = MirrorProtocol.objects.filter(is_download=True) elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol: @@ -102,23 +129,17 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, ip_version |= Q(has_ipv6=True) qset = qset.filter(ip_version) + if smart_protocol: + urls = default_protocol_filter(qset) + else: + urls = qset + if not use_status: - urls = qset.order_by('mirror__name', 'url') - urls = sorted(urls, key=attrgetter('real_country')) + sort_key = attrgetter('real_country', 'mirror.name', 'url') + urls = sorted(urls, key=sort_key) template = 'mirrors/mirrorlist.txt' else: - status_info = get_mirror_statuses() - scores = dict([(u.id, u.score) for u in status_info['urls']]) - urls = [] - for u in qset: - u.score = scores.get(u.id, None) - # also include mirrors that don't have an up to date score - # (as opposed to those that have been set with no score) - if (u.id not in scores) or \ - (u.score and u.score < 100.0): - urls.append(u) - # if a url doesn't have a score, treat it as the highest possible - urls = sorted(urls, key=lambda x: x.score or 100.0) + urls = status_filter(urls) template = 'mirrors/mirrorlist_status.txt' return direct_to_template(request, template, { @@ -128,6 +149,10 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, def find_mirrors_simple(request, protocol): + if protocol == 'smart': + # generate a 'smart' mirrorlist, one that only includes FTP mirrors if + # no HTTP mirror is available in that country. + return find_mirrors(request, smart_protocol=True) proto = get_object_or_404(MirrorProtocol, protocol=protocol) return find_mirrors(request, protocols=[proto]) -- cgit v1.2.3-2-g168b From a5f5557493446bede78adb0584c88208234f874e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 12 May 2012 09:32:30 -0500 Subject: Use python json module directly in place of simplejson As of Python 2.6, this is a builtin module that has all the same functions and capabilities of the Django simplejson module. Additionally simplejson is deprecated in the upcoming Django 1.5 release. Signed-off-by: Dan McGee --- mirrors/views.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index eac78ff2..b0be6238 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -1,5 +1,6 @@ from datetime import timedelta from itertools import groupby +import json from operator import attrgetter, itemgetter from django import forms @@ -10,7 +11,6 @@ from django.http import Http404, HttpResponse from django.shortcuts import get_object_or_404 from django.views.decorators.csrf import csrf_exempt from django.views.generic.simple import direct_to_template -from django.utils import simplejson from django_countries.countries import COUNTRIES from .models import Mirror, MirrorUrl, MirrorProtocol @@ -237,8 +237,7 @@ def status_json(request): status_info = get_mirror_statuses() data = status_info.copy() data['version'] = 3 - to_json = simplejson.dumps(data, ensure_ascii=False, - cls=MirrorStatusJSONEncoder) + to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder) response = HttpResponse(to_json, mimetype='application/json') return response -- cgit v1.2.3-2-g168b From ae1c526ffbe908322f0dd8d8805360b81ab22b0f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 13 May 2012 20:35:50 -0500 Subject: Add ability to restrict status report to single tier This should make it easier to catch errors in our Tier 1 mirrors. Signed-off-by: Dan McGee --- mirrors/views.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index b0be6238..8f092be7 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -13,7 +13,7 @@ from django.views.decorators.csrf import csrf_exempt from django.views.generic.simple import direct_to_template from django_countries.countries import COUNTRIES -from .models import Mirror, MirrorUrl, MirrorProtocol +from .models import Mirror, MirrorUrl, MirrorProtocol, TIER_CHOICES from .utils import get_mirror_statuses, get_mirror_errors COUNTRY_LOOKUP = dict(COUNTRIES) @@ -184,7 +184,11 @@ def mirror_details(request, name): {'mirror': mirror, 'urls': all_urls}) -def status(request): +def status(request, tier=None): + if tier is not None: + tier = int(tier) + if tier not in [t[0] for t in TIER_CHOICES]: + raise Http404 bad_timedelta = timedelta(days=3) status_info = get_mirror_statuses() @@ -192,17 +196,26 @@ def status(request): good_urls = [] bad_urls = [] for url in urls: + # screen by tier if we were asked to + if tier is not None and url.mirror.tier != tier: + continue # split them into good and bad lists based on delay if not url.delay or url.delay > bad_timedelta: bad_urls.append(url) else: good_urls.append(url) + error_logs = get_mirror_errors() + if tier is not None: + error_logs = [log for log in error_logs + if log['url__mirror__tier'] == tier] + context = status_info.copy() context.update({ 'good_urls': sorted(good_urls, key=attrgetter('score')), 'bad_urls': sorted(bad_urls, key=lambda u: u.delay or timedelta.max), - 'error_logs': get_mirror_errors(), + 'error_logs': error_logs, + 'tier': tier, }) return direct_to_template(request, 'mirrors/status.html', context) -- cgit v1.2.3-2-g168b From 76c37ce3acc7a4af0271c7535d4a33042f7749b5 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 24 Jul 2012 09:35:55 -0500 Subject: Replace deprecated direct_to_template() with render() shortcut Now that Django actually provides a concise way to use a RequestContext object without instantiating it, we can use that rather than the old function-based generic view that worked well to do the same. Additionally, these function-based generic views will be gone in Django 1.5, so might as well make the move now. Signed-off-by: Dan McGee --- mirrors/views.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index 8f092be7..400c084d 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -8,9 +8,8 @@ from django.forms.widgets import CheckboxSelectMultiple from django.core.serializers.json import DjangoJSONEncoder from django.db.models import Q from django.http import Http404, HttpResponse -from django.shortcuts import get_object_or_404 +from django.shortcuts import get_object_or_404, render from django.views.decorators.csrf import csrf_exempt -from django.views.generic.simple import direct_to_template from django_countries.countries import COUNTRIES from .models import Mirror, MirrorUrl, MirrorProtocol, TIER_CHOICES @@ -76,7 +75,7 @@ def generate_mirrorlist(request): else: form = MirrorlistForm() - return direct_to_template(request, 'mirrors/mirrorlist_generate.html', + return render(request, 'mirrors/mirrorlist_generate.html', {'mirrorlist_form': form}) @@ -142,10 +141,10 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, urls = status_filter(urls) template = 'mirrors/mirrorlist_status.txt' - return direct_to_template(request, template, { - 'mirror_urls': urls, - }, - mimetype='text/plain') + context = { + 'mirror_urls': urls, + } + return render(request, template, context, content_type='text/plain') def find_mirrors_simple(request, protocol): @@ -161,7 +160,7 @@ def mirrors(request): mirror_list = Mirror.objects.select_related().order_by('tier', 'country') if not request.user.is_authenticated(): mirror_list = mirror_list.filter(public=True, active=True) - return direct_to_template(request, 'mirrors/mirrors.html', + return render(request, 'mirrors/mirrors.html', {'mirror_list': mirror_list}) @@ -180,7 +179,7 @@ def mirror_details(request, name): all_urls = set(checked_urls).union(all_urls) all_urls = sorted(all_urls, key=attrgetter('url')) - return direct_to_template(request, 'mirrors/mirror_details.html', + return render(request, 'mirrors/mirror_details.html', {'mirror': mirror, 'urls': all_urls}) @@ -217,7 +216,7 @@ def status(request, tier=None): 'error_logs': error_logs, 'tier': tier, }) - return direct_to_template(request, 'mirrors/status.html', context) + return render(request, 'mirrors/status.html', context) class MirrorStatusJSONEncoder(DjangoJSONEncoder): -- cgit v1.2.3-2-g168b From 686942b8788fa43031b3999ac00d60baadc82f53 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 31 Jul 2012 19:56:49 -0500 Subject: Declare 'enums' at class scope Signed-off-by: Dan McGee --- mirrors/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index 400c084d..2c2577f4 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -12,7 +12,7 @@ from django.shortcuts import get_object_or_404, render from django.views.decorators.csrf import csrf_exempt from django_countries.countries import COUNTRIES -from .models import Mirror, MirrorUrl, MirrorProtocol, TIER_CHOICES +from .models import Mirror, MirrorUrl, MirrorProtocol from .utils import get_mirror_statuses, get_mirror_errors COUNTRY_LOOKUP = dict(COUNTRIES) @@ -186,7 +186,7 @@ def mirror_details(request, name): def status(request, tier=None): if tier is not None: tier = int(tier) - if tier not in [t[0] for t in TIER_CHOICES]: + if tier not in [t[0] for t in Mirror.TIER_CHOICES]: raise Http404 bad_timedelta = timedelta(days=3) status_info = get_mirror_statuses() -- cgit v1.2.3-2-g168b From e9c4985538c067a09a186967f77c5395fb60675b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 18 Sep 2012 22:10:54 -0500 Subject: Sort mirrorlist by English country name, not ISO code Fixes FS#31503. Signed-off-by: Dan McGee --- mirrors/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index 2c2577f4..11719223 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -134,7 +134,7 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, urls = qset if not use_status: - sort_key = attrgetter('real_country', 'mirror.name', 'url') + sort_key = attrgetter('real_country.name', 'mirror.name', 'url') urls = sorted(urls, key=sort_key) template = 'mirrors/mirrorlist.txt' else: -- cgit v1.2.3-2-g168b From 4ab5d6947795f1fef0d38601ec7ad3ca5f62173e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 10 Nov 2012 14:13:34 -0600 Subject: Add mirror extended status JSON view When asking for status for a single mirror, we can include logs from the past 24 hours in addition to the normal information we provide. This is slated for usage by a frontend graph still to come, similar to those on the NTP pool website. Signed-off-by: Dan McGee --- mirrors/views.py | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index 11719223..cbd86611 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -9,10 +9,11 @@ from django.core.serializers.json import DjangoJSONEncoder from django.db.models import Q from django.http import Http404, HttpResponse from django.shortcuts import get_object_or_404, render +from django.utils.timezone import now from django.views.decorators.csrf import csrf_exempt from django_countries.countries import COUNTRIES -from .models import Mirror, MirrorUrl, MirrorProtocol +from .models import Mirror, MirrorUrl, MirrorProtocol, MirrorLog from .utils import get_mirror_statuses, get_mirror_errors COUNTRY_LOOKUP = dict(COUNTRIES) @@ -183,6 +184,19 @@ def mirror_details(request, name): {'mirror': mirror, 'urls': all_urls}) +def mirror_details_json(request, name): + mirror = get_object_or_404(Mirror, name=name) + status_info = get_mirror_statuses() + data = status_info.copy() + data['version'] = 3 + # include only URLs for this particular mirror + data['urls'] = [url for url in data['urls'] if url.mirror_id == mirror.id] + to_json = json.dumps(data, ensure_ascii=False, + cls=ExtendedMirrorStatusJSONEncoder) + response = HttpResponse(to_json, mimetype='application/json') + return response + + def status(request, tier=None): if tier is not None: tier = int(tier) @@ -222,8 +236,8 @@ def status(request, tier=None): class MirrorStatusJSONEncoder(DjangoJSONEncoder): '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl serialization. The base class takes care of datetime.datetime types.''' - url_attributes = ['url', 'protocol', 'last_sync', 'completion_pct', - 'delay', 'duration_avg', 'duration_stddev', 'score'] + url_attributes = ('url', 'protocol', 'last_sync', 'completion_pct', + 'delay', 'duration_avg', 'duration_stddev', 'score') def default(self, obj): if isinstance(obj, timedelta): @@ -245,6 +259,23 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder): return super(MirrorStatusJSONEncoder, self).default(obj) +class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): + '''Adds URL check history information.''' + log_attributes = ('check_time', 'last_sync', 'duration', 'is_success') + + def default(self, obj): + if isinstance(obj, MirrorUrl): + data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj) + cutoff = now() - timedelta(hours=24) + data['logs'] = obj.logs.filter(check_time__gte=cutoff) + return data + if isinstance(obj, MirrorLog): + data = dict((attr, getattr(obj, attr)) + for attr in self.log_attributes) + return data + return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) + + def status_json(request): status_info = get_mirror_statuses() data = status_info.copy() -- cgit v1.2.3-2-g168b From 86102c6e645451c03e3e576060eba7f93350bf6b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 10 Nov 2012 14:19:23 -0600 Subject: Allow filtering retrieved mirror statuses by mirror_id When we don't need them all, no need to fetch them all. Let the database do the work for us, hopefully. Signed-off-by: Dan McGee --- mirrors/views.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index cbd86611..4b9721dc 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -14,7 +14,7 @@ from django.views.decorators.csrf import csrf_exempt from django_countries.countries import COUNTRIES from .models import Mirror, MirrorUrl, MirrorProtocol, MirrorLog -from .utils import get_mirror_statuses, get_mirror_errors +from .utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF COUNTRY_LOOKUP = dict(COUNTRIES) @@ -171,7 +171,7 @@ def mirror_details(request, name): (not mirror.public or not mirror.active): raise Http404 - status_info = get_mirror_statuses() + status_info = get_mirror_statuses(mirror_ids=[mirror.id]) checked_urls = [url for url in status_info['urls'] \ if url.mirror_id == mirror.id] all_urls = mirror.urls.select_related('protocol') @@ -186,7 +186,7 @@ def mirror_details(request, name): def mirror_details_json(request, name): mirror = get_object_or_404(Mirror, name=name) - status_info = get_mirror_statuses() + status_info = get_mirror_statuses(mirror_ids=[mirror.id]) data = status_info.copy() data['version'] = 3 # include only URLs for this particular mirror @@ -266,7 +266,7 @@ class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): def default(self, obj): if isinstance(obj, MirrorUrl): data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj) - cutoff = now() - timedelta(hours=24) + cutoff = now() - DEFAULT_CUTOFF data['logs'] = obj.logs.filter(check_time__gte=cutoff) return data if isinstance(obj, MirrorLog): -- cgit v1.2.3-2-g168b From a358e132886972dc4e9f1f546e36a5f3a2218a39 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 10 Nov 2012 17:09:51 -0600 Subject: Mirror status graph, now with points AND lines We might have to tweak the interpolation method once we see this with real data, but for now it looks really pretty locally. Signed-off-by: Dan McGee --- mirrors/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index 4b9721dc..be01e919 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -267,7 +267,8 @@ class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): if isinstance(obj, MirrorUrl): data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj) cutoff = now() - DEFAULT_CUTOFF - data['logs'] = obj.logs.filter(check_time__gte=cutoff) + data['logs'] = obj.logs.filter( + check_time__gte=cutoff).order_by('check_time') return data if isinstance(obj, MirrorLog): data = dict((attr, getattr(obj, attr)) -- cgit v1.2.3-2-g168b From e26d5722289bd2e972633891d8dac09296b0cbc4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 11 Nov 2012 14:55:37 -0600 Subject: Mirror graph tweaking after usage with real data * Clamp y-axis minimum to 0. * Don't plot `is_success == false` values. * Ensure URLs are sorted predictably. Signed-off-by: Dan McGee --- mirrors/views.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index be01e919..5e374b4d 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -189,8 +189,6 @@ def mirror_details_json(request, name): status_info = get_mirror_statuses(mirror_ids=[mirror.id]) data = status_info.copy() data['version'] = 3 - # include only URLs for this particular mirror - data['urls'] = [url for url in data['urls'] if url.mirror_id == mirror.id] to_json = json.dumps(data, ensure_ascii=False, cls=ExtendedMirrorStatusJSONEncoder) response = HttpResponse(to_json, mimetype='application/json') -- cgit v1.2.3-2-g168b From 92837c93acc66056391dd0b98515b89f8fc49691 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 12 Nov 2012 21:37:08 -0600 Subject: Prefetch the available protocols on the mirror overview page Otherwise we are doing one query per mirror, which at this point is over 100 separate queries. Signed-off-by: Dan McGee --- mirrors/views.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index 5e374b4d..2e1e83b6 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -159,8 +159,16 @@ def find_mirrors_simple(request, protocol): def mirrors(request): mirror_list = Mirror.objects.select_related().order_by('tier', 'country') + protos = MirrorUrl.objects.values_list( + 'mirror_id', 'protocol__protocol').order_by( + 'mirror__id', 'protocol__protocol') if not request.user.is_authenticated(): mirror_list = mirror_list.filter(public=True, active=True) + protos = protos.filter(mirror__public=True, mirror__active=True) + protos = dict((k, list(v)) for k, v in groupby(protos, key=itemgetter(0))) + for mirror in mirror_list: + items = protos.get(mirror.id, []) + mirror.protocols = [item[1] for item in items] return render(request, 'mirrors/mirrors.html', {'mirror_list': mirror_list}) -- cgit v1.2.3-2-g168b From 6dd4d54bb0adbbb0f8c2b1beaa92b7a58971cf88 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 16 Nov 2012 16:20:11 -0600 Subject: Use Python 2.7 dictionary comprehension syntax Rather than the old idiom of dict((k, v) for <> in <>). Signed-off-by: Dan McGee --- mirrors/views.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index 2e1e83b6..d0ce0a97 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -94,7 +94,7 @@ def default_protocol_filter(original_urls): def status_filter(original_urls): status_info = get_mirror_statuses() - scores = dict((u.id, u.score) for u in status_info['urls']) + scores = {u.id: u.score for u in status_info['urls']} urls = [] for u in original_urls: u.score = scores.get(u.id, None) @@ -165,7 +165,7 @@ def mirrors(request): if not request.user.is_authenticated(): mirror_list = mirror_list.filter(public=True, active=True) protos = protos.filter(mirror__public=True, mirror__active=True) - protos = dict((k, list(v)) for k, v in groupby(protos, key=itemgetter(0))) + protos = {k: list(v) for k, v in groupby(protos, key=itemgetter(0))} for mirror in mirror_list: items = protos.get(mirror.id, []) mirror.protocols = [item[1] for item in items] @@ -253,8 +253,7 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder): # mainly for queryset serialization return list(obj) if isinstance(obj, MirrorUrl): - data = dict((attr, getattr(obj, attr)) - for attr in self.url_attributes) + data = {attr: getattr(obj, attr) for attr in self.url_attributes} # get any override on the country attribute first country = obj.real_country data['country'] = unicode(country.name) @@ -277,9 +276,7 @@ class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): check_time__gte=cutoff).order_by('check_time') return data if isinstance(obj, MirrorLog): - data = dict((attr, getattr(obj, attr)) - for attr in self.log_attributes) - return data + return {attr: getattr(obj, attr) for attr in self.log_attributes} return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) -- cgit v1.2.3-2-g168b From 2b68963ad1049f4b0198bed6da16e385aedc119e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 27 Dec 2012 16:37:49 -0600 Subject: Ensure mirror protocols are distinct Signed-off-by: Dan McGee --- mirrors/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index d0ce0a97..22da631a 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -161,7 +161,7 @@ def mirrors(request): mirror_list = Mirror.objects.select_related().order_by('tier', 'country') protos = MirrorUrl.objects.values_list( 'mirror_id', 'protocol__protocol').order_by( - 'mirror__id', 'protocol__protocol') + 'mirror__id', 'protocol__protocol').distinct() if not request.user.is_authenticated(): mirror_list = mirror_list.filter(public=True, active=True) protos = protos.filter(mirror__public=True, mirror__active=True) -- cgit v1.2.3-2-g168b From 66850026ca934e5a09238e9033c541cdc5085a42 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 13 Jan 2013 22:34:33 -0600 Subject: Use content_type and not mimetype on HttpResponse() Bug #16519 in Django deprecates mimetype, so update our code accordingly. Signed-off-by: Dan McGee --- mirrors/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index 22da631a..d3867802 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -199,7 +199,7 @@ def mirror_details_json(request, name): data['version'] = 3 to_json = json.dumps(data, ensure_ascii=False, cls=ExtendedMirrorStatusJSONEncoder) - response = HttpResponse(to_json, mimetype='application/json') + response = HttpResponse(to_json, content_type='application/json') return response @@ -285,7 +285,7 @@ def status_json(request): data = status_info.copy() data['version'] = 3 to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder) - response = HttpResponse(to_json, mimetype='application/json') + response = HttpResponse(to_json, content_type='application/json') return response # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From 6f0ae6746baea657ee6d7c21ac0813a04f825443 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 14 Jan 2013 01:00:11 -0600 Subject: Drop country column from mirror table We now always look for this information at the URL level, not the mirror level. This simplifies quite a bit of code in and around the mirror views. Signed-off-by: Dan McGee --- mirrors/views.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index d3867802..545e3557 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -42,9 +42,6 @@ class MirrorlistForm(forms.Form): def get_countries(self): country_codes = set() - country_codes.update(Mirror.objects.filter(active=True).exclude( - country='').values_list( - 'country', flat=True).order_by().distinct()) country_codes.update(MirrorUrl.objects.filter( mirror__active=True).exclude(country='').values_list( 'country', flat=True).order_by().distinct()) @@ -81,7 +78,7 @@ def generate_mirrorlist(request): def default_protocol_filter(original_urls): - key_func = attrgetter('real_country') + key_func = attrgetter('country') sorted_urls = sorted(original_urls, key=key_func) urls = [] for _, group in groupby(sorted_urls, key=key_func): @@ -119,8 +116,7 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, protocol__in=protocols, mirror__public=True, mirror__active=True) if countries and 'all' not in countries: - qset = qset.filter(Q(country__in=countries) | - Q(mirror__country__in=countries)) + qset = qset.filter(country__in=countries) ip_version = Q() if ipv4_supported: @@ -135,7 +131,7 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, urls = qset if not use_status: - sort_key = attrgetter('real_country.name', 'mirror.name', 'url') + sort_key = attrgetter('country.name', 'mirror.name', 'url') urls = sorted(urls, key=sort_key) template = 'mirrors/mirrorlist.txt' else: @@ -158,7 +154,7 @@ def find_mirrors_simple(request, protocol): def mirrors(request): - mirror_list = Mirror.objects.select_related().order_by('tier', 'country') + mirror_list = Mirror.objects.select_related().order_by('tier', 'name') protos = MirrorUrl.objects.values_list( 'mirror_id', 'protocol__protocol').order_by( 'mirror__id', 'protocol__protocol').distinct() @@ -254,8 +250,7 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder): return list(obj) if isinstance(obj, MirrorUrl): data = {attr: getattr(obj, attr) for attr in self.url_attributes} - # get any override on the country attribute first - country = obj.real_country + country = obj.country data['country'] = unicode(country.name) data['country_code'] = country.code return data -- cgit v1.2.3-2-g168b From ff6db38f1dc6ed1eb53454a7e16615ec1ad76d7a Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 14 Jan 2013 01:27:34 -0600 Subject: Ensure URLs without check data work on mirror details page Less noticeable in production as the templates don't show '@@@INVALID@@@' there, but we were trying to access attributes that don't actually exist on certain mirror objects. Signed-off-by: Dan McGee --- mirrors/views.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index 545e3557..30df5472 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -176,13 +176,17 @@ def mirror_details(request, name): raise Http404 status_info = get_mirror_statuses(mirror_ids=[mirror.id]) - checked_urls = [url for url in status_info['urls'] \ - if url.mirror_id == mirror.id] - all_urls = mirror.urls.select_related('protocol') - # get each item from checked_urls and supplement with anything in all_urls - # if it wasn't there - all_urls = set(checked_urls).union(all_urls) - all_urls = sorted(all_urls, key=attrgetter('url')) + checked_urls = {url for url in status_info['urls'] \ + if url.mirror_id == mirror.id} + all_urls = set(mirror.urls.select_related('protocol')) + # Add dummy data for URLs that we haven't checked recently + other_urls = all_urls.difference(checked_urls) + print other_urls + for url in other_urls: + for attr in ('last_sync', 'completion_pct', 'delay', 'duration_avg', + 'duration_stddev', 'score'): + setattr(url, attr, None) + all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url')) return render(request, 'mirrors/mirror_details.html', {'mirror': mirror, 'urls': all_urls}) -- cgit v1.2.3-2-g168b From 0f6a0a1cd0011c8ad137a4b27d0b39a7e1129fb7 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 14 Jan 2013 08:54:33 -0600 Subject: Support mirror status JSON by tier Just as we do for the normal status HTML view. Signed-off-by: Dan McGee --- mirrors/views.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index 30df5472..c0ed6670 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -279,9 +279,15 @@ class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) -def status_json(request): +def status_json(request, tier=None): + if tier is not None: + tier = int(tier) + if tier not in [t[0] for t in Mirror.TIER_CHOICES]: + raise Http404 status_info = get_mirror_statuses() data = status_info.copy() + if tier is not None: + data['urls'] = [url for url in data['urls'] if url.mirror.tier == tier] data['version'] = 3 to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder) response = HttpResponse(to_json, content_type='application/json') -- cgit v1.2.3-2-g168b From 1f9aef78f39c90191eddf2233c278086a15052de Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 16 Jan 2013 00:18:26 -0600 Subject: Remove debugging print statement Signed-off-by: Dan McGee --- mirrors/views.py | 1 - 1 file changed, 1 deletion(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index c0ed6670..56397633 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -181,7 +181,6 @@ def mirror_details(request, name): all_urls = set(mirror.urls.select_related('protocol')) # Add dummy data for URLs that we haven't checked recently other_urls = all_urls.difference(checked_urls) - print other_urls for url in other_urls: for attr in ('last_sync', 'completion_pct', 'delay', 'duration_avg', 'duration_stddev', 'score'): -- cgit v1.2.3-2-g168b From c588d1c85f86f5ee10a96bec679111c8675b703c Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 13 Apr 2013 11:38:11 -0500 Subject: Support only a single mirror ID in error/status retrieval This simplifies things and makes injecting this single mirror ID into custom SQL a whole lot easier. Signed-off-by: Dan McGee --- mirrors/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index 56397633..07e28d40 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -175,7 +175,7 @@ def mirror_details(request, name): (not mirror.public or not mirror.active): raise Http404 - status_info = get_mirror_statuses(mirror_ids=[mirror.id]) + status_info = get_mirror_statuses(mirror_id=mirror.id) checked_urls = {url for url in status_info['urls'] \ if url.mirror_id == mirror.id} all_urls = set(mirror.urls.select_related('protocol')) @@ -193,7 +193,7 @@ def mirror_details(request, name): def mirror_details_json(request, name): mirror = get_object_or_404(Mirror, name=name) - status_info = get_mirror_statuses(mirror_ids=[mirror.id]) + status_info = get_mirror_statuses(mirror_id=mirror.id) data = status_info.copy() data['version'] = 3 to_json = json.dumps(data, ensure_ascii=False, -- cgit v1.2.3-2-g168b From 0cad22a5eca20ecb64b04d0912592ea6a5361e0d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 14 Apr 2013 13:59:49 -0500 Subject: Add a JSON view for retrieving mirror check locations Signed-off-by: Dan McGee --- mirrors/views.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index 07e28d40..30f96b63 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -13,7 +13,8 @@ from django.utils.timezone import now from django.views.decorators.csrf import csrf_exempt from django_countries.countries import COUNTRIES -from .models import Mirror, MirrorUrl, MirrorProtocol, MirrorLog +from .models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog, + CheckLocation) from .utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF COUNTRY_LOOKUP = dict(COUNTRIES) @@ -264,7 +265,8 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder): class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): '''Adds URL check history information.''' - log_attributes = ('check_time', 'last_sync', 'duration', 'is_success') + log_attributes = ('check_time', 'last_sync', 'duration', 'is_success', + 'location_id') def default(self, obj): if isinstance(obj, MirrorUrl): @@ -292,4 +294,31 @@ def status_json(request, tier=None): response = HttpResponse(to_json, content_type='application/json') return response + +class LocationJSONEncoder(DjangoJSONEncoder): + '''Base JSONEncoder extended to handle CheckLocation objects.''' + + def default(self, obj): + if hasattr(obj, '__iter__'): + # mainly for queryset serialization + return list(obj) + if isinstance(obj, CheckLocation): + return { + 'hostname': obj.hostname, + 'source_ip': obj.source_ip, + 'country': unicode(obj.country.name), + 'country_code': obj.country.code, + 'ip_version': obj.ip_version, + } + return super(LocationJSONEncoder, self).default(obj) + + +def locations_json(request): + data = {} + data['version'] = 1 + data['locations'] = CheckLocation.objects.all() + to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From 9b07cb1ebdc8c5cc5dff66a7edb02e0ddc9f4733 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 14 Apr 2013 15:08:25 -0500 Subject: Draw one mirror status graph per check location Rather than lump it all together and have odd spikes depending on which side of the Atlantic checked a mirror in a given timeslot, draw a chart per check location. Signed-off-by: Dan McGee --- mirrors/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index 30f96b63..9311fb8f 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -304,6 +304,7 @@ class LocationJSONEncoder(DjangoJSONEncoder): return list(obj) if isinstance(obj, CheckLocation): return { + 'id': obj.pk, 'hostname': obj.hostname, 'source_ip': obj.source_ip, 'country': unicode(obj.country.name), @@ -316,7 +317,7 @@ class LocationJSONEncoder(DjangoJSONEncoder): def locations_json(request): data = {} data['version'] = 1 - data['locations'] = CheckLocation.objects.all() + data['locations'] = CheckLocation.objects.all().order_by('pk') to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder) response = HttpResponse(to_json, content_type='application/json') return response -- cgit v1.2.3-2-g168b From 0589853360d12a1746e2d8e92e798f2727a0b5df Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 16 Apr 2013 21:52:20 -0500 Subject: Remove COUNTRY_LOOKUP global variable This is only used in one place, so it makes more sense for it to not be globally accessible. Signed-off-by: Dan McGee --- mirrors/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index 9311fb8f..73d40297 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -17,8 +17,6 @@ from .models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog, CheckLocation) from .utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF -COUNTRY_LOOKUP = dict(COUNTRIES) - class MirrorlistForm(forms.Form): country = forms.MultipleChoiceField(required=False) @@ -29,6 +27,8 @@ class MirrorlistForm(forms.Form): widget=CheckboxSelectMultiple) use_mirror_status = forms.BooleanField(required=False) + countries = dict(COUNTRIES) + def __init__(self, *args, **kwargs): super(MirrorlistForm, self).__init__(*args, **kwargs) fields = self.fields @@ -46,7 +46,7 @@ class MirrorlistForm(forms.Form): country_codes.update(MirrorUrl.objects.filter( mirror__active=True).exclude(country='').values_list( 'country', flat=True).order_by().distinct()) - countries = [(code, COUNTRY_LOOKUP[code]) for code in country_codes] + countries = [(code, self.countries[code]) for code in country_codes] return sorted(countries, key=itemgetter(1)) def as_div(self): -- cgit v1.2.3-2-g168b