Merge branch 'archweb-generic2'

Conflicts:
	README.BRANDING
	local_settings.py.example
	packages/templatetags/package_extras.py
	public/views.py
	releng/views.py
	settings.py
	sitestatic/archnavbar/archnavbar.css
	sitestatic/silhouette.png
	templates/base.html
	templates/packages/differences.html
	templates/packages/opensearch.xml
	templates/packages/search.html
	templates/public/donate.html
	templates/public/download.html
	templates/public/feeds.html
	templates/public/index.html
	urls.py

1 files changed, 176 insertions, 63 deletions

diff --git a/mirrors/views.py b/mirrors/views.py
index c52656f7..73d40297 100644
--- a/mirrors/views.py
+++ b/mirrors/views.py
@@ -1,4 +1,6 @@
 from datetime import timedelta
+from itertools import groupby
+import json
 from operator import attrgetter, itemgetter
 
 from django import forms
@@ -6,16 +8,14 @@ from django.forms.widgets import CheckboxSelectMultiple
 from django.core.serializers.json import DjangoJSONEncoder
 from django.db.models import Q
 from django.http import Http404, HttpResponse
-from django.shortcuts import get_object_or_404
+from django.shortcuts import get_object_or_404, render
+from django.utils.timezone import now
 from django.views.decorators.csrf import csrf_exempt
-from django.views.generic.simple import direct_to_template
-from django.utils import simplejson
 from django_countries.countries import COUNTRIES
 
-from .models import Mirror, MirrorUrl, MirrorProtocol
-from .utils import get_mirror_statuses, get_mirror_errors
-
-COUNTRY_LOOKUP = dict(COUNTRIES)
+from .models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog,
+        CheckLocation)
+from .utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF
 
 
 class MirrorlistForm(forms.Form):
@@ -27,6 +27,8 @@ class MirrorlistForm(forms.Form):
             widget=CheckboxSelectMultiple)
     use_mirror_status = forms.BooleanField(required=False)
 
+    countries = dict(COUNTRIES)
+
     def __init__(self, *args, **kwargs):
         super(MirrorlistForm, self).__init__(*args, **kwargs)
         fields = self.fields
@@ -41,13 +43,10 @@ class MirrorlistForm(forms.Form):
 
     def get_countries(self):
         country_codes = set()
-        country_codes.update(Mirror.objects.filter(active=True).exclude(
-                country='').values_list(
-                'country', flat=True).order_by().distinct())
         country_codes.update(MirrorUrl.objects.filter(
                 mirror__active=True).exclude(country='').values_list(
                 'country', flat=True).order_by().distinct())
-        countries = [(code, COUNTRY_LOOKUP[code]) for code in country_codes]
+        countries = [(code, self.countries[code]) for code in country_codes]
         return sorted(countries, key=itemgetter(1))
 
     def as_div(self):
@@ -75,22 +74,50 @@ def generate_mirrorlist(request):
     else:
         form = MirrorlistForm()
 
-    return direct_to_template(request, 'mirrors/mirrorlist_generate.html',
+    return render(request, 'mirrors/mirrorlist_generate.html',
             {'mirrorlist_form': form})
 
 
+def default_protocol_filter(original_urls):
+    key_func = attrgetter('country')
+    sorted_urls = sorted(original_urls, key=key_func)
+    urls = []
+    for _, group in groupby(sorted_urls, key=key_func):
+        cntry_urls = list(group)
+        if any(url.protocol.default for url in cntry_urls):
+            cntry_urls = [url for url in cntry_urls if url.protocol.default]
+        urls.extend(cntry_urls)
+    return urls
+
+
+def status_filter(original_urls):
+    status_info = get_mirror_statuses()
+    scores = {u.id: u.score for u in status_info['urls']}
+    urls = []
+    for u in original_urls:
+        u.score = scores.get(u.id, None)
+        # also include mirrors that don't have an up to date score
+        # (as opposed to those that have been set with no score)
+        if (u.id not in scores) or (u.score and u.score < 100.0):
+            urls.append(u)
+    # if a url doesn't have a score, treat it as the highest possible
+    return sorted(urls, key=lambda x: x.score or 100.0)
+
+
 def find_mirrors(request, countries=None, protocols=None, use_status=False,
-        ipv4_supported=True, ipv6_supported=True):
+        ipv4_supported=True, ipv6_supported=True, smart_protocol=False):
     if not protocols:
-        protocols = MirrorProtocol.objects.filter(
-                is_download=True).values_list('protocol', flat=True)
+        protocols = MirrorProtocol.objects.filter(is_download=True)
+    elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol:
+        # we already have a queryset, no need to query again
+        pass
+    else:
+        protocols = MirrorProtocol.objects.filter(protocol__in=protocols)
     qset = MirrorUrl.objects.select_related().filter(
-            protocol__protocol__in=protocols,
-            mirror__public=True, mirror__active=True,
-    )
+            protocol__in=protocols,
+            mirror__public=True, mirror__active=True)
     if countries and 'all' not in countries:
-        qset = qset.filter(Q(country__in=countries) |
-                Q(mirror__country__in=countries))
+        qset = qset.filter(country__in=countries)
 
     ip_version = Q()
     if ipv4_supported:
@@ -99,36 +126,47 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False,
         ip_version |= Q(has_ipv6=True)
     qset = qset.filter(ip_version)
 
+    if smart_protocol:
+        urls = default_protocol_filter(qset)
+    else:
+        urls = qset
+
     if not use_status:
-        urls = qset.order_by('mirror__name', 'url')
-        urls = sorted(urls, key=attrgetter('real_country'))
+        sort_key = attrgetter('country.name', 'mirror.name', 'url')
+        urls = sorted(urls, key=sort_key)
         template = 'mirrors/mirrorlist.txt'
     else:
-        status_info = get_mirror_statuses()
-        scores = dict([(u.id, u.score) for u in status_info['urls']])
-        urls = []
-        for u in qset:
-            u.score = scores.get(u.id, None)
-            # also include mirrors that don't have an up to date score
-            # (as opposed to those that have been set with no score)
-            if (u.id not in scores) or \
-                    (u.score and u.score < 100.0):
-                urls.append(u)
-        # if a url doesn't have a score, treat it as the highest possible
-        urls = sorted(urls, key=lambda x: x.score or 100.0)
+        urls = status_filter(urls)
         template = 'mirrors/mirrorlist_status.txt'
 
-    return direct_to_template(request, template, {
-                'mirror_urls': urls,
-            },
-            mimetype='text/plain')
+    context = {
+        'mirror_urls': urls,
+    }
+    return render(request, template, context, content_type='text/plain')
+
+
+def find_mirrors_simple(request, protocol):
+    if protocol == 'smart':
+        # generate a 'smart' mirrorlist, one that only includes FTP mirrors if
+        # no HTTP mirror is available in that country.
+        return find_mirrors(request, smart_protocol=True)
+    proto = get_object_or_404(MirrorProtocol, protocol=protocol)
+    return find_mirrors(request, protocols=[proto])
 
 
 def mirrors(request):
-    mirror_list = Mirror.objects.select_related().order_by('tier', 'country')
+    mirror_list = Mirror.objects.select_related().order_by('tier', 'name')
+    protos = MirrorUrl.objects.values_list(
+            'mirror_id', 'protocol__protocol').order_by(
+            'mirror__id', 'protocol__protocol').distinct()
     if not request.user.is_authenticated():
         mirror_list = mirror_list.filter(public=True, active=True)
-    return direct_to_template(request, 'mirrors/mirrors.html',
+        protos = protos.filter(mirror__public=True, mirror__active=True)
+    protos = {k: list(v) for k, v in groupby(protos, key=itemgetter(0))}
+    for mirror in mirror_list:
+        items = protos.get(mirror.id, [])
+        mirror.protocols = [item[1] for item in items]
+    return render(request, 'mirrors/mirrors.html',
             {'mirror_list': mirror_list})
 
 
@@ -138,20 +176,38 @@ def mirror_details(request, name):
             (not mirror.public or not mirror.active):
         raise Http404
 
-    status_info = get_mirror_statuses()
-    checked_urls = [url for url in status_info['urls'] \
-            if url.mirror_id == mirror.id]
-    all_urls = mirror.urls.select_related('protocol')
-    # get each item from checked_urls and supplement with anything in all_urls
-    # if it wasn't there
-    all_urls = set(checked_urls).union(all_urls)
-    all_urls = sorted(all_urls, key=attrgetter('url'))
-
-    return direct_to_template(request, 'mirrors/mirror_details.html',
+    status_info = get_mirror_statuses(mirror_id=mirror.id)
+    checked_urls = {url for url in status_info['urls'] \
+            if url.mirror_id == mirror.id}
+    all_urls = set(mirror.urls.select_related('protocol'))
+    # Add dummy data for URLs that we haven't checked recently
+    other_urls = all_urls.difference(checked_urls)
+    for url in other_urls:
+        for attr in ('last_sync', 'completion_pct', 'delay', 'duration_avg',
+                'duration_stddev', 'score'):
+            setattr(url, attr, None)
+    all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url'))
+
+    return render(request, 'mirrors/mirror_details.html',
             {'mirror': mirror, 'urls': all_urls})
 
 
-def status(request):
+def mirror_details_json(request, name):
+    mirror = get_object_or_404(Mirror, name=name)
+    status_info = get_mirror_statuses(mirror_id=mirror.id)
+    data = status_info.copy()
+    data['version'] = 3
+    to_json = json.dumps(data, ensure_ascii=False,
+            cls=ExtendedMirrorStatusJSONEncoder)
+    response = HttpResponse(to_json, content_type='application/json')
+    return response
+
+
+def status(request, tier=None):
+    if tier is not None:
+        tier = int(tier)
+        if tier not in [t[0] for t in Mirror.TIER_CHOICES]:
+            raise Http404
     bad_timedelta = timedelta(days=3)
     status_info = get_mirror_statuses()
 
@@ -159,26 +215,35 @@ def status(request):
     good_urls = []
     bad_urls = []
     for url in urls:
+        # screen by tier if we were asked to
+        if tier is not None and url.mirror.tier != tier:
+            continue
         # split them into good and bad lists based on delay
         if not url.delay or url.delay > bad_timedelta:
             bad_urls.append(url)
         else:
             good_urls.append(url)
 
+    error_logs = get_mirror_errors()
+    if tier is not None:
+        error_logs = [log for log in error_logs
+                if log['url__mirror__tier'] == tier]
+
     context = status_info.copy()
     context.update({
         'good_urls': sorted(good_urls, key=attrgetter('score')),
         'bad_urls': sorted(bad_urls, key=lambda u: u.delay or timedelta.max),
-        'error_logs': get_mirror_errors(),
+        'error_logs': error_logs,
+        'tier': tier,
     })
-    return direct_to_template(request, 'mirrors/status.html', context)
+    return render(request, 'mirrors/status.html', context)
 
 
 class MirrorStatusJSONEncoder(DjangoJSONEncoder):
     '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl
     serialization. The base class takes care of datetime.datetime types.'''
-    url_attributes = ['url', 'protocol', 'last_sync', 'completion_pct',
-            'delay', 'duration_avg', 'duration_stddev', 'score']
+    url_attributes = ('url', 'protocol', 'last_sync', 'completion_pct',
+            'delay', 'duration_avg', 'duration_stddev', 'score')
 
     def default(self, obj):
         if isinstance(obj, timedelta):
@@ -188,10 +253,8 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder):
             # mainly for queryset serialization
             return list(obj)
         if isinstance(obj, MirrorUrl):
-            data = dict((attr, getattr(obj, attr))
-                    for attr in self.url_attributes)
-            # get any override on the country attribute first
-            country = obj.real_country
+            data = {attr: getattr(obj, attr) for attr in self.url_attributes}
+            country = obj.country
             data['country'] = unicode(country.name)
             data['country_code'] = country.code
             return data
@@ -200,13 +263,63 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder):
         return super(MirrorStatusJSONEncoder, self).default(obj)
 
 
-def status_json(request):
+class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder):
+    '''Adds URL check history information.'''
+    log_attributes = ('check_time', 'last_sync', 'duration', 'is_success',
+            'location_id')
+
+    def default(self, obj):
+        if isinstance(obj, MirrorUrl):
+            data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj)
+            cutoff = now() - DEFAULT_CUTOFF
+            data['logs'] = obj.logs.filter(
+                    check_time__gte=cutoff).order_by('check_time')
+            return data
+        if isinstance(obj, MirrorLog):
+            return {attr: getattr(obj, attr) for attr in self.log_attributes}
+        return super(ExtendedMirrorStatusJSONEncoder, self).default(obj)
+
+
+def status_json(request, tier=None):
+    if tier is not None:
+        tier = int(tier)
+        if tier not in [t[0] for t in Mirror.TIER_CHOICES]:
+            raise Http404
     status_info = get_mirror_statuses()
     data = status_info.copy()
+    if tier is not None:
+        data['urls'] = [url for url in data['urls'] if url.mirror.tier == tier]
     data['version'] = 3
-    to_json = simplejson.dumps(data, ensure_ascii=False,
-            cls=MirrorStatusJSONEncoder)
-    response = HttpResponse(to_json, mimetype='application/json')
+    to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder)
+    response = HttpResponse(to_json, content_type='application/json')
+    return response
+
+
+class LocationJSONEncoder(DjangoJSONEncoder):
+    '''Base JSONEncoder extended to handle CheckLocation objects.'''
+
+    def default(self, obj):
+        if hasattr(obj, '__iter__'):
+            # mainly for queryset serialization
+            return list(obj)
+        if isinstance(obj, CheckLocation):
+            return {
+                'id': obj.pk,
+                'hostname': obj.hostname,
+                'source_ip': obj.source_ip,
+                'country': unicode(obj.country.name),
+                'country_code': obj.country.code,
+                'ip_version': obj.ip_version,
+            }
+        return super(LocationJSONEncoder, self).default(obj)
+
+
+def locations_json(request):
+    data = {}
+    data['version'] = 1
+    data['locations'] = CheckLocation.objects.all().order_by('pk')
+    to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder)
+    response = HttpResponse(to_json, content_type='application/json')
     return response
 
 # vim: set ts=4 sw=4 et: