Merge git://projects.archlinux.org/archweb

Conflicts:
	templates/public/index.html
	urls.py

4 files changed, 61 insertions, 18 deletions

diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py
index bab79219..d7aad4e2 100644
--- a/mirrors/management/commands/mirrorcheck.py
+++ b/mirrors/management/commands/mirrorcheck.py
@@ -10,7 +10,6 @@ Usage: ./manage.py mirrorcheck
 """
 
 from django.core.management.base import NoArgsCommand
-from django.db.models import Q
 
 from datetime import datetime, timedelta
 import logging
@@ -51,10 +50,10 @@ class Command(NoArgsCommand):
 
         return check_current_mirrors()
 
-def parse_rfc3339_datetime(time):
+def parse_rfc3339_datetime(time_string):
     # '2010-09-02 11:05:06+02:00'
     m = re.match('^(\d{4})-(\d{2})-(\d{2}) '
-            '(\d{2}):(\d{2}):(\d{2})([-+])(\d{2}):(\d{2})', time)
+            '(\d{2}):(\d{2}):(\d{2})([-+])(\d{2}):(\d{2})', time_string)
     if m:
         vals = m.groups()
         parsed = datetime(int(vals[0]), int(vals[1]), int(vals[2]),
@@ -123,46 +122,55 @@ def check_mirror_url(mirror_url):
         log.error = "Connection timed out."
         logger.debug("failed: %s, %s" % (url, log.error))
 
-    log.save()
     return log
 
-def mirror_url_worker(queue):
+def mirror_url_worker(work, output):
     while True:
         try:
-            item = queue.get(block=False)
+            item = work.get(block=False)
             try:
-                check_mirror_url(item)
+                log = check_mirror_url(item)
+                output.put(log)
             finally:
-                queue.task_done()
+                work.task_done()
         except Empty:
             return 0
 
 class MirrorCheckPool(object):
     def __init__(self, work, num_threads=10):
         self.tasks = Queue()
-        for i in work:
+        self.logs = Queue()
+        for i in list(work):
             self.tasks.put(i)
         self.threads = []
         for i in range(num_threads):
-            thread = Thread(target=mirror_url_worker, args=(self.tasks,))
+            thread = Thread(target=mirror_url_worker,
+                    args=(self.tasks, self.logs))
             thread.daemon = True
             self.threads.append(thread)
 
-    def run_and_join(self):
+    def run(self):
         logger.debug("starting threads")
         for t in self.threads:
             t.start()
         logger.debug("joining on all threads")
         self.tasks.join()
+        logger.debug("processing log entries")
+        try:
+            while True:
+                log = self.logs.get(block=False)
+                log.save()
+                self.logs.task_done()
+        except Empty:
+            logger.debug("all log items saved to database")
 
 def check_current_mirrors():
     urls = MirrorUrl.objects.filter(
-            Q(protocol__protocol__iexact='HTTP') |
-            Q(protocol__protocol__iexact='FTP'),
+            protocol__is_download=True,
             mirror__active=True, mirror__public=True)
 
     pool = MirrorCheckPool(urls)
-    pool.run_and_join()
+    pool.run()
     return 0
 
 # For lack of a better place to put it, here is a query to get latest check
diff --git a/mirrors/models.py b/mirrors/models.py
index f1286026..e070b1cd 100644
--- a/mirrors/models.py
+++ b/mirrors/models.py
@@ -80,7 +80,7 @@ class MirrorLog(models.Model):
     error = models.CharField(max_length=255, blank=True, default='')
 
     def __unicode__(self):
-        return "Check of %s at %s" % (url.url, check_time)
+        return "Check of %s at %s" % (self.url.url, self.check_time)
 
     class Meta:
         verbose_name = 'Mirror Check Log'
diff --git a/mirrors/utils.py b/mirrors/utils.py
index c64d53c5..124b66e6 100644
--- a/mirrors/utils.py
+++ b/mirrors/utils.py
@@ -51,7 +51,7 @@ def get_mirror_statuses(cutoff=default_cutoff):
         check_frequency = None
 
     for url in urls:
-        url.completion_pct = float(url.success_count) / num_checks
+        url.completion_pct = float(url.success_count) / url.check_count
         if url.id in delays:
             url_delays = delays[url.id]
             d = sum(url_delays, datetime.timedelta()) / len(url_delays)
diff --git a/mirrors/views.py b/mirrors/views.py
index fb7d3361..a2b94de8 100644
--- a/mirrors/views.py
+++ b/mirrors/views.py
@@ -1,10 +1,12 @@
 from django import forms
+from django.core.serializers.json import DjangoJSONEncoder
 from django.db.models import Avg, Count, Max, Min, StdDev
 from django.db.models import Q
-from django.http import Http404
+from django.http import Http404, HttpResponse
 from django.shortcuts import get_object_or_404
 from django.views.decorators.csrf import csrf_exempt
 from django.views.generic.simple import direct_to_template
+from django.utils import simplejson
 
 from main.utils import make_choice
 from .models import Mirror, MirrorUrl, MirrorProtocol
@@ -34,7 +36,7 @@ class MirrorlistForm(forms.Form):
 
 @csrf_exempt
 def generate_mirrorlist(request):
-    if request.REQUEST.get('country', ''):
+    if request.method == 'POST' or len(request.GET) > 0:
         form = MirrorlistForm(data=request.REQUEST)
         if form.is_valid():
             countries = form.cleaned_data['country']
@@ -129,4 +131,37 @@ def status(request):
     })
     return direct_to_template(request, 'mirrors/status.html', context)
 
+class MirrorStatusJSONEncoder(DjangoJSONEncoder):
+    '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl
+    serialization. The base class takes care of datetime.datetime types.'''
+    url_attributes = ['url', 'protocol', 'last_sync', 'completion_pct',
+            'delay', 'duration_avg', 'duration_stddev', 'score']
+
+    def default(self, obj):
+        if isinstance(obj, datetime.timedelta):
+            # always returned as integer seconds
+            return obj.days * 24 * 3600 + obj.seconds
+        if hasattr(obj, '__iter__'):
+            # mainly for queryset serialization
+            return list(obj)
+        if isinstance(obj, MirrorUrl):
+            data = {}
+            for attr in self.url_attributes:
+                data[attr] = getattr(obj, attr)
+            # separate because it isn't on the URL directly
+            data['country'] = obj.mirror.country
+            return data
+        if isinstance(obj, MirrorProtocol):
+            return unicode(obj)
+        return super(MirrorStatusJSONEncoder, self).default(obj)
+
+def status_json(request):
+    status_info = get_mirror_statuses()
+    data = status_info.copy()
+    data['version'] = 1
+    to_json = simplejson.dumps(data, ensure_ascii=False,
+            cls=MirrorStatusJSONEncoder)
+    response = HttpResponse(to_json, mimetype='application/json')
+    return response
+
 # vim: set ts=4 sw=4 et: