From 17b6ce186c6e793417c73e955bfc01b4f4b96864 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 26 Feb 2015 19:16:44 -0600 Subject: Move mirrors views into subdirectory We'll start splitting these up as we did in packages/. Signed-off-by: Dan McGee --- mirrors/views/__init__.py | 368 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 368 insertions(+) create mode 100644 mirrors/views/__init__.py (limited to 'mirrors/views/__init__.py') diff --git a/mirrors/views/__init__.py b/mirrors/views/__init__.py new file mode 100644 index 00000000..54c427b3 --- /dev/null +++ b/mirrors/views/__init__.py @@ -0,0 +1,368 @@ +from datetime import timedelta +from itertools import groupby +import json +from operator import attrgetter, itemgetter + +from django import forms +from django.forms.widgets import SelectMultiple, CheckboxSelectMultiple +from django.core.serializers.json import DjangoJSONEncoder +from django.db import connection +from django.db.models import Q +from django.http import Http404, HttpResponse +from django.shortcuts import get_object_or_404, redirect, render +from django.utils.timezone import now +from django.views.decorators.cache import cache_page +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import condition +from django_countries import countries +from django_countries.fields import Country + +from ..models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog, + CheckLocation) +from ..utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF + + +class MirrorlistForm(forms.Form): + country = forms.MultipleChoiceField(required=False, + widget=SelectMultiple(attrs={'size': '12'})) + protocol = forms.MultipleChoiceField(required=False, + widget=CheckboxSelectMultiple) + ip_version = forms.MultipleChoiceField(required=False, + label="IP version", choices=(('4','IPv4'), ('6','IPv6')), + widget=CheckboxSelectMultiple) + use_mirror_status = forms.BooleanField(required=False) + + def __init__(self, *args, **kwargs): + super(MirrorlistForm, self).__init__(*args, **kwargs) + fields = self.fields + fields['country'].choices = [('all','All')] + self.get_countries() + fields['country'].initial = ['all'] + protos = [(p.protocol, p.protocol) for p in + MirrorProtocol.objects.filter(is_download=True)] + initial = MirrorProtocol.objects.filter(is_download=True, default=True) + fields['protocol'].choices = protos + fields['protocol'].initial = [p.protocol for p in initial] + fields['ip_version'].initial = ['4'] + + def get_countries(self): + country_codes = set() + country_codes.update(MirrorUrl.objects.filter(active=True, + mirror__active=True).exclude(country='').values_list( + 'country', flat=True).order_by().distinct()) + code_list = [(code, countries.name(code)) for code in country_codes] + return sorted(code_list, key=itemgetter(1)) + + def as_div(self): + "Returns this form rendered as HTML s." + return self._html_output( + normal_row = u'%(label)s %(field)s%(help_text)s', + error_row = u'%s', + row_ender = '', + help_text_html = u' %s', + errors_on_separate_row = True) + + +@csrf_exempt +def generate_mirrorlist(request): + if request.method == 'POST' or len(request.GET) > 0: + form = MirrorlistForm(data=request.REQUEST) + if form.is_valid(): + countries = form.cleaned_data['country'] + protocols = form.cleaned_data['protocol'] + use_status = form.cleaned_data['use_mirror_status'] + ipv4 = '4' in form.cleaned_data['ip_version'] + ipv6 = '6' in form.cleaned_data['ip_version'] + return find_mirrors(request, countries, protocols, + use_status, ipv4, ipv6) + else: + form = MirrorlistForm() + + return render(request, 'mirrors/mirrorlist_generate.html', + {'mirrorlist_form': form}) + + +def status_filter(original_urls): + status_info = get_mirror_statuses() + scores = {u.id: u.score for u in status_info['urls']} + urls = [] + for u in original_urls: + u.score = scores.get(u.id, None) + # also include mirrors that don't have an up to date score + # (as opposed to those that have been set with no score) + if (u.id not in scores) or (u.score and u.score < 100.0): + urls.append(u) + # if a url doesn't have a score, treat it as the highest possible + return sorted(urls, key=lambda x: x.score or 100.0) + + +def find_mirrors(request, countries=None, protocols=None, use_status=False, + ipv4_supported=True, ipv6_supported=True): + if not protocols: + protocols = MirrorProtocol.objects.filter(is_download=True) + elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol: + # we already have a queryset, no need to query again + pass + else: + protocols = MirrorProtocol.objects.filter(protocol__in=protocols) + qset = MirrorUrl.objects.select_related().filter( + protocol__in=protocols, active=True, + mirror__public=True, mirror__active=True) + if countries and 'all' not in countries: + qset = qset.filter(country__in=countries) + + ip_version = Q() + if ipv4_supported: + ip_version |= Q(has_ipv4=True) + if ipv6_supported: + ip_version |= Q(has_ipv6=True) + qset = qset.filter(ip_version) + + if not use_status: + sort_key = attrgetter('country.name', 'mirror.name', 'url') + urls = sorted(qset, key=sort_key) + template = 'mirrors/mirrorlist.txt' + else: + urls = status_filter(qset) + template = 'mirrors/mirrorlist_status.txt' + + context = { + 'mirror_urls': urls, + } + return render(request, template, context, content_type='text/plain') + + +def find_mirrors_simple(request, protocol): + if protocol == 'smart': + return redirect('mirrorlist_simple', 'http', permanent=True) + proto = get_object_or_404(MirrorProtocol, protocol=protocol) + return find_mirrors(request, protocols=[proto]) + + +def mirrors(request): + mirror_list = Mirror.objects.select_related().order_by('tier', 'name') + protos = MirrorUrl.objects.values_list( + 'mirror_id', 'protocol__protocol').order_by( + 'mirror_id', 'protocol__protocol').distinct() + countries = MirrorUrl.objects.values_list( + 'mirror_id', 'country').order_by( + 'mirror_id', 'country').distinct() + + if not request.user.is_authenticated(): + mirror_list = mirror_list.filter(public=True, active=True) + protos = protos.filter( + mirror__public=True, mirror__active=True, active=True) + countries = countries.filter( + mirror__public=True, mirror__active=True, active=True) + + protos = {k: list(v) for k, v in groupby(protos, key=itemgetter(0))} + countries = {k: list(v) for k, v in groupby(countries, key=itemgetter(0))} + + for mirror in mirror_list: + item_protos = protos.get(mirror.id, []) + mirror.protocols = [item[1] for item in item_protos] + mirror.country = None + item_countries = countries.get(mirror.id, []) + if len(item_countries) == 1: + mirror.country = Country(item_countries[0][1]) + + return render(request, 'mirrors/mirrors.html', + {'mirror_list': mirror_list}) + + +def mirror_details(request, name): + mirror = get_object_or_404(Mirror, name=name) + authorized = request.user.is_authenticated() + if not authorized and \ + (not mirror.public or not mirror.active): + raise Http404 + error_cutoff = timedelta(days=7) + + status_info = get_mirror_statuses(mirror_id=mirror.id, + show_all=authorized) + checked_urls = {url for url in status_info['urls'] \ + if url.mirror_id == mirror.id} + all_urls = mirror.urls.select_related('protocol') + if not authorized: + all_urls = all_urls.filter(active=True) + all_urls = set(all_urls) + # Add dummy data for URLs that we haven't checked recently + other_urls = all_urls.difference(checked_urls) + for url in other_urls: + for attr in ('last_sync', 'completion_pct', 'delay', 'duration_avg', + 'duration_stddev', 'score'): + setattr(url, attr, None) + all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url')) + + error_logs = get_mirror_errors(mirror_id=mirror.id, cutoff=error_cutoff, + show_all=True) + + context = { + 'mirror': mirror, + 'urls': all_urls, + 'cutoff': error_cutoff, + 'error_logs': error_logs, + } + return render(request, 'mirrors/mirror_details.html', context) + + +def mirror_details_json(request, name): + authorized = request.user.is_authenticated() + mirror = get_object_or_404(Mirror, name=name) + if not authorized and (not mirror.public or not mirror.active): + raise Http404 + status_info = get_mirror_statuses(mirror_id=mirror.id, + show_all=authorized) + data = status_info.copy() + data['version'] = 3 + to_json = json.dumps(data, ensure_ascii=False, + cls=ExtendedMirrorStatusJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + + +def url_details(request, name, url_id): + url = get_object_or_404(MirrorUrl.objects.select_related(), + id=url_id, mirror__name=name) + mirror = url.mirror + authorized = request.user.is_authenticated() + if not authorized and \ + (not mirror.public or not mirror.active or not url.active): + raise Http404 + error_cutoff = timedelta(days=7) + cutoff_time = now() - error_cutoff + logs = MirrorLog.objects.select_related('location').filter( + url=url, check_time__gte=cutoff_time).order_by('-check_time') + + context = { + 'url': url, + 'logs': logs, + } + return render(request, 'mirrors/url_details.html', context) + + +def status_last_modified(request, *args, **kwargs): + cursor = connection.cursor() + cursor.execute("SELECT MAX(check_time) FROM mirrors_mirrorlog") + return cursor.fetchone()[0] + + +@condition(last_modified_func=status_last_modified) +def status(request, tier=None): + if tier is not None: + tier = int(tier) + if tier not in [t[0] for t in Mirror.TIER_CHOICES]: + raise Http404 + bad_timedelta = timedelta(days=3) + status_info = get_mirror_statuses() + + urls = status_info['urls'] + good_urls = [] + bad_urls = [] + for url in urls: + # screen by tier if we were asked to + if tier is not None and url.mirror.tier != tier: + continue + # split them into good and bad lists based on delay + if url.completion_pct is None: + # skip URLs that have never been checked + continue + elif not url.delay or url.delay > bad_timedelta: + bad_urls.append(url) + else: + good_urls.append(url) + + error_logs = get_mirror_errors() + if tier is not None: + error_logs = [log for log in error_logs + if log['url'].mirror.tier == tier] + + context = status_info.copy() + context.update({ + 'good_urls': sorted(good_urls, key=attrgetter('score')), + 'bad_urls': sorted(bad_urls, key=lambda u: u.delay or timedelta.max), + 'error_logs': error_logs, + 'tier': tier, + }) + return render(request, 'mirrors/status.html', context) + + +class MirrorStatusJSONEncoder(DjangoJSONEncoder): + '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl + serialization. The base class takes care of datetime.datetime types.''' + url_attributes = ('url', 'protocol', 'last_sync', 'completion_pct', + 'delay', 'duration_avg', 'duration_stddev', 'score') + + def default(self, obj): + if isinstance(obj, timedelta): + # always returned as integer seconds + return obj.days * 24 * 3600 + obj.seconds + if isinstance(obj, MirrorUrl): + data = {attr: getattr(obj, attr) for attr in self.url_attributes} + country = obj.country + data['country'] = unicode(country.name) + data['country_code'] = country.code + return data + if isinstance(obj, MirrorProtocol): + return unicode(obj) + return super(MirrorStatusJSONEncoder, self).default(obj) + + +class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): + '''Adds URL check history information.''' + log_attributes = ('check_time', 'last_sync', 'duration', 'is_success', + 'location_id') + + def default(self, obj): + if isinstance(obj, MirrorUrl): + data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj) + cutoff = now() - DEFAULT_CUTOFF + data['logs'] = list(obj.logs.filter( + check_time__gte=cutoff).order_by('check_time')) + return data + if isinstance(obj, MirrorLog): + return {attr: getattr(obj, attr) for attr in self.log_attributes} + return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) + + +@cache_page(67) +@condition(last_modified_func=status_last_modified) +def status_json(request, tier=None): + if tier is not None: + tier = int(tier) + if tier not in [t[0] for t in Mirror.TIER_CHOICES]: + raise Http404 + status_info = get_mirror_statuses() + data = status_info.copy() + if tier is not None: + data['urls'] = [url for url in data['urls'] if url.mirror.tier == tier] + data['version'] = 3 + to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + + +class LocationJSONEncoder(DjangoJSONEncoder): + '''Base JSONEncoder extended to handle CheckLocation objects.''' + + def default(self, obj): + if isinstance(obj, CheckLocation): + return { + 'id': obj.pk, + 'hostname': obj.hostname, + 'source_ip': obj.source_ip, + 'country': unicode(obj.country.name), + 'country_code': obj.country.code, + 'ip_version': obj.ip_version, + } + return super(LocationJSONEncoder, self).default(obj) + + +def locations_json(request): + data = {} + data['version'] = 1 + data['locations'] = list(CheckLocation.objects.all().order_by('pk')) + to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder) + response = HttpResponse(to_json, content_type='application/json') + return response + +# vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b