diff options
Diffstat (limited to 'mirrors/views.py')
| -rw-r--r-- | mirrors/views.py | 239 |
1 files changed, 176 insertions, 63 deletions
diff --git a/mirrors/views.py b/mirrors/views.py index c52656f7..73d40297 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -1,4 +1,6 @@ from datetime import timedelta +from itertools import groupby +import json from operator import attrgetter, itemgetter from django import forms @@ -6,16 +8,14 @@ from django.forms.widgets import CheckboxSelectMultiple from django.core.serializers.json import DjangoJSONEncoder from django.db.models import Q from django.http import Http404, HttpResponse -from django.shortcuts import get_object_or_404 +from django.shortcuts import get_object_or_404, render +from django.utils.timezone import now from django.views.decorators.csrf import csrf_exempt -from django.views.generic.simple import direct_to_template -from django.utils import simplejson from django_countries.countries import COUNTRIES -from .models import Mirror, MirrorUrl, MirrorProtocol -from .utils import get_mirror_statuses, get_mirror_errors - -COUNTRY_LOOKUP = dict(COUNTRIES) +from .models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog, + CheckLocation) +from .utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF class MirrorlistForm(forms.Form): @@ -27,6 +27,8 @@ class MirrorlistForm(forms.Form): widget=CheckboxSelectMultiple) use_mirror_status = forms.BooleanField(required=False) + countries = dict(COUNTRIES) + def __init__(self, *args, **kwargs): super(MirrorlistForm, self).__init__(*args, **kwargs) fields = self.fields @@ -41,13 +43,10 @@ class MirrorlistForm(forms.Form): def get_countries(self): country_codes = set() - country_codes.update(Mirror.objects.filter(active=True).exclude( - country='').values_list( - 'country', flat=True).order_by().distinct()) country_codes.update(MirrorUrl.objects.filter( mirror__active=True).exclude(country='').values_list( 'country', flat=True).order_by().distinct()) - countries = [(code, COUNTRY_LOOKUP[code]) for code in country_codes] + countries = [(code, self.countries[code]) for code in country_codes] return sorted(countries, key=itemgetter(1)) def as_div(self): @@ -75,22 +74,50 @@ def generate_mirrorlist(request): else: form = MirrorlistForm() - return direct_to_template(request, 'mirrors/mirrorlist_generate.html', + return render(request, 'mirrors/mirrorlist_generate.html', {'mirrorlist_form': form}) +def default_protocol_filter(original_urls): + key_func = attrgetter('country') + sorted_urls = sorted(original_urls, key=key_func) + urls = [] + for _, group in groupby(sorted_urls, key=key_func): + cntry_urls = list(group) + if any(url.protocol.default for url in cntry_urls): + cntry_urls = [url for url in cntry_urls if url.protocol.default] + urls.extend(cntry_urls) + return urls + + +def status_filter(original_urls): + status_info = get_mirror_statuses() + scores = {u.id: u.score for u in status_info['urls']} + urls = [] + for u in original_urls: + u.score = scores.get(u.id, None) + # also include mirrors that don't have an up to date score + # (as opposed to those that have been set with no score) + if (u.id not in scores) or (u.score and u.score < 100.0): + urls.append(u) + # if a url doesn't have a score, treat it as the highest possible + return sorted(urls, key=lambda x: x.score or 100.0) + + def find_mirrors(request, countries=None, protocols=None, use_status=False, - ipv4_supported=True, ipv6_supported=True): + ipv4_supported=True, ipv6_supported=True, smart_protocol=False): if not protocols: - protocols = MirrorProtocol.objects.filter( - is_download=True).values_list('protocol', flat=True) + protocols = MirrorProtocol.objects.filter(is_download=True) + elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol: + # we already have a queryset, no need to query again + pass + else: + protocols = MirrorProtocol.objects.filter(protocol__in=protocols) qset = MirrorUrl.objects.select_related().filter( - protocol__protocol__in=protocols, - mirror__public=True, mirror__active=True, - ) + protocol__in=protocols, + mirror__public=True, mirror__active=True) if countries and 'all' not in countries: - qset = qset.filter(Q(country__in=countries) | - Q(mirror__country__in=countries)) + qset = qset.filter(country__in=countries) ip_version = Q() if ipv4_supported: @@ -99,36 +126,47 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, ip_version |= Q(has_ipv6=True) qset = qset.filter(ip_version) + if smart_protocol: + urls = default_protocol_filter(qset) + else: + urls = qset + if not use_status: - urls = qset.order_by('mirror__name', 'url') - urls = sorted(urls, key=attrgetter('real_country')) + sort_key = attrgetter('country.name', 'mirror.name', 'url') + urls = sorted(urls, key=sort_key) template = 'mirrors/mirrorlist.txt' else: - status_info = get_mirror_statuses() - scores = dict([(u.id, u.score) for u in status_info['urls']]) - urls = [] - for u in qset: - u.score = scores.get(u.id, None) - # also include mirrors that don't have an up to date score - # (as opposed to those that have been set with no score) - if (u.id not in scores) or \ - (u.score and u.score < 100.0): - urls.append(u) - # if a url doesn't have a score, treat it as the highest possible - urls = sorted(urls, key=lambda x: x.score or 100.0) + urls = status_filter(urls) template = 'mirrors/mirrorlist_status.txt' - return direct_to_template(request, template, { - 'mirror_urls': urls, - }, - mimetype='text/plain') + context = { + 'mirror_urls': urls, + } + return render(request, template, context, content_type='text/plain') + + +def find_mirrors_simple(request, protocol): + if protocol == 'smart': + # generate a 'smart' mirrorlist, one that only includes FTP mirrors if + # no HTTP mirror is available in that country. + return find_mirrors(request, smart_protocol=True) + proto = get_object_or_404(MirrorProtocol, protocol=protocol) + return find_mirrors(request, protocols=[proto]) def mirrors(request): - mirror_list = Mirror.objects.select_related().order_by('tier', 'country') + mirror_list = Mirror.objects.select_related().order_by('tier', 'name') + protos = MirrorUrl.objects.values_list( + 'mirror_id', 'protocol__protocol').order_by( + 'mirror__id', 'protocol__protocol').distinct() if not request.user.is_authenticated(): mirror_list = mirror_list.filter(public=True, active=True) - return direct_to_template(request, 'mirrors/mirrors.html', + protos = protos.filter(mirror__public=True, mirror__active=True) + protos = {k: list(v) for k, v in groupby(protos, key=itemgetter(0))} + for mirror in mirror_list: + items = protos.get(mirror.id, []) + mirror.protocols = [item[1] for item in items] + return render(request, 'mirrors/mirrors.html', {'mirror_list': mirror_list}) @@ -138,20 +176,38 @@ def mirror_details(request, name): (not mirror.public or not mirror.active): raise Http404 - status_info = get_mirror_statuses() - checked_urls = [url for url in status_info['urls'] \ - if url.mirror_id == mirror.id] - all_urls = mirror.urls.select_related('protocol') - # get each item from checked_urls and supplement with anything in all_urls - # if it wasn't there - all_urls = set(checked_urls).union(all_urls) - all_urls = sorted(all_urls, key=attrgetter('url')) - - return direct_to_template(request, 'mirrors/mirror_details.html', + status_info = get_mirror_statuses(mirror_id=mirror.id) + checked_urls = {url for url in status_info['urls'] \ + if url.mirror_id == mirror.id} + all_urls = set(mirror.urls.select_related('protocol')) + # Add dummy data for URLs that we haven't checked recently + other_urls = all_urls.difference(checked_urls) + for url in other_urls: + for attr in ('last_sync', 'completion_pct', 'delay', 'duration_avg', + 'duration_stddev', 'score'): + setattr(url, attr, None) + all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url')) + + return render(request, 'mirrors/mirror_details.html', {'mirror': mirror, 'urls': all_urls}) -def status(request): +def mirror_details_json(request, name): + mirror = get_object_or_404(Mirror, name=name) + status_info = get_mirror_statuses(mirror_id=mirror.id) + data = status_info.copy() + data['version'] = 3 + to_json = json.dumps(data, ensure_ascii=False, + cls=ExtendedMirrorStatusJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + + +def status(request, tier=None): + if tier is not None: + tier = int(tier) + if tier not in [t[0] for t in Mirror.TIER_CHOICES]: + raise Http404 bad_timedelta = timedelta(days=3) status_info = get_mirror_statuses() @@ -159,26 +215,35 @@ def status(request): good_urls = [] bad_urls = [] for url in urls: + # screen by tier if we were asked to + if tier is not None and url.mirror.tier != tier: + continue # split them into good and bad lists based on delay if not url.delay or url.delay > bad_timedelta: bad_urls.append(url) else: good_urls.append(url) + error_logs = get_mirror_errors() + if tier is not None: + error_logs = [log for log in error_logs + if log['url__mirror__tier'] == tier] + context = status_info.copy() context.update({ 'good_urls': sorted(good_urls, key=attrgetter('score')), 'bad_urls': sorted(bad_urls, key=lambda u: u.delay or timedelta.max), - 'error_logs': get_mirror_errors(), + 'error_logs': error_logs, + 'tier': tier, }) - return direct_to_template(request, 'mirrors/status.html', context) + return render(request, 'mirrors/status.html', context) class MirrorStatusJSONEncoder(DjangoJSONEncoder): '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl serialization. The base class takes care of datetime.datetime types.''' - url_attributes = ['url', 'protocol', 'last_sync', 'completion_pct', - 'delay', 'duration_avg', 'duration_stddev', 'score'] + url_attributes = ('url', 'protocol', 'last_sync', 'completion_pct', + 'delay', 'duration_avg', 'duration_stddev', 'score') def default(self, obj): if isinstance(obj, timedelta): @@ -188,10 +253,8 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder): # mainly for queryset serialization return list(obj) if isinstance(obj, MirrorUrl): - data = dict((attr, getattr(obj, attr)) - for attr in self.url_attributes) - # get any override on the country attribute first - country = obj.real_country + data = {attr: getattr(obj, attr) for attr in self.url_attributes} + country = obj.country data['country'] = unicode(country.name) data['country_code'] = country.code return data @@ -200,13 +263,63 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder): return super(MirrorStatusJSONEncoder, self).default(obj) -def status_json(request): +class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): + '''Adds URL check history information.''' + log_attributes = ('check_time', 'last_sync', 'duration', 'is_success', + 'location_id') + + def default(self, obj): + if isinstance(obj, MirrorUrl): + data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj) + cutoff = now() - DEFAULT_CUTOFF + data['logs'] = obj.logs.filter( + check_time__gte=cutoff).order_by('check_time') + return data + if isinstance(obj, MirrorLog): + return {attr: getattr(obj, attr) for attr in self.log_attributes} + return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) + + +def status_json(request, tier=None): + if tier is not None: + tier = int(tier) + if tier not in [t[0] for t in Mirror.TIER_CHOICES]: + raise Http404 status_info = get_mirror_statuses() data = status_info.copy() + if tier is not None: + data['urls'] = [url for url in data['urls'] if url.mirror.tier == tier] data['version'] = 3 - to_json = simplejson.dumps(data, ensure_ascii=False, - cls=MirrorStatusJSONEncoder) - response = HttpResponse(to_json, mimetype='application/json') + to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + + +class LocationJSONEncoder(DjangoJSONEncoder): + '''Base JSONEncoder extended to handle CheckLocation objects.''' + + def default(self, obj): + if hasattr(obj, '__iter__'): + # mainly for queryset serialization + return list(obj) + if isinstance(obj, CheckLocation): + return { + 'id': obj.pk, + 'hostname': obj.hostname, + 'source_ip': obj.source_ip, + 'country': unicode(obj.country.name), + 'country_code': obj.country.code, + 'ip_version': obj.ip_version, + } + return super(LocationJSONEncoder, self).default(obj) + + +def locations_json(request): + data = {} + data['version'] = 1 + data['locations'] = CheckLocation.objects.all().order_by('pk') + to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') return response # vim: set ts=4 sw=4 et: |