diff options
Diffstat (limited to 'mirrors')
| -rw-r--r-- | mirrors/management/commands/mirrorcheck.py | 36 | ||||
| -rw-r--r-- | mirrors/models.py | 2 | ||||
| -rw-r--r-- | mirrors/utils.py | 2 | ||||
| -rw-r--r-- | mirrors/views.py | 39 |
4 files changed, 61 insertions, 18 deletions
diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index bab79219..d7aad4e2 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -10,7 +10,6 @@ Usage: ./manage.py mirrorcheck """ from django.core.management.base import NoArgsCommand -from django.db.models import Q from datetime import datetime, timedelta import logging @@ -51,10 +50,10 @@ class Command(NoArgsCommand): return check_current_mirrors() -def parse_rfc3339_datetime(time): +def parse_rfc3339_datetime(time_string): # '2010-09-02 11:05:06+02:00' m = re.match('^(\d{4})-(\d{2})-(\d{2}) ' - '(\d{2}):(\d{2}):(\d{2})([-+])(\d{2}):(\d{2})', time) + '(\d{2}):(\d{2}):(\d{2})([-+])(\d{2}):(\d{2})', time_string) if m: vals = m.groups() parsed = datetime(int(vals[0]), int(vals[1]), int(vals[2]), @@ -123,46 +122,55 @@ def check_mirror_url(mirror_url): log.error = "Connection timed out." logger.debug("failed: %s, %s" % (url, log.error)) - log.save() return log -def mirror_url_worker(queue): +def mirror_url_worker(work, output): while True: try: - item = queue.get(block=False) + item = work.get(block=False) try: - check_mirror_url(item) + log = check_mirror_url(item) + output.put(log) finally: - queue.task_done() + work.task_done() except Empty: return 0 class MirrorCheckPool(object): def __init__(self, work, num_threads=10): self.tasks = Queue() - for i in work: + self.logs = Queue() + for i in list(work): self.tasks.put(i) self.threads = [] for i in range(num_threads): - thread = Thread(target=mirror_url_worker, args=(self.tasks,)) + thread = Thread(target=mirror_url_worker, + args=(self.tasks, self.logs)) thread.daemon = True self.threads.append(thread) - def run_and_join(self): + def run(self): logger.debug("starting threads") for t in self.threads: t.start() logger.debug("joining on all threads") self.tasks.join() + logger.debug("processing log entries") + try: + while True: + log = self.logs.get(block=False) + log.save() + self.logs.task_done() + except Empty: + logger.debug("all log items saved to database") def check_current_mirrors(): urls = MirrorUrl.objects.filter( - Q(protocol__protocol__iexact='HTTP') | - Q(protocol__protocol__iexact='FTP'), + protocol__is_download=True, mirror__active=True, mirror__public=True) pool = MirrorCheckPool(urls) - pool.run_and_join() + pool.run() return 0 # For lack of a better place to put it, here is a query to get latest check diff --git a/mirrors/models.py b/mirrors/models.py index f1286026..e070b1cd 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -80,7 +80,7 @@ class MirrorLog(models.Model): error = models.CharField(max_length=255, blank=True, default='') def __unicode__(self): - return "Check of %s at %s" % (url.url, check_time) + return "Check of %s at %s" % (self.url.url, self.check_time) class Meta: verbose_name = 'Mirror Check Log' diff --git a/mirrors/utils.py b/mirrors/utils.py index c64d53c5..124b66e6 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -51,7 +51,7 @@ def get_mirror_statuses(cutoff=default_cutoff): check_frequency = None for url in urls: - url.completion_pct = float(url.success_count) / num_checks + url.completion_pct = float(url.success_count) / url.check_count if url.id in delays: url_delays = delays[url.id] d = sum(url_delays, datetime.timedelta()) / len(url_delays) diff --git a/mirrors/views.py b/mirrors/views.py index fb7d3361..a2b94de8 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -1,10 +1,12 @@ from django import forms +from django.core.serializers.json import DjangoJSONEncoder from django.db.models import Avg, Count, Max, Min, StdDev from django.db.models import Q -from django.http import Http404 +from django.http import Http404, HttpResponse from django.shortcuts import get_object_or_404 from django.views.decorators.csrf import csrf_exempt from django.views.generic.simple import direct_to_template +from django.utils import simplejson from main.utils import make_choice from .models import Mirror, MirrorUrl, MirrorProtocol @@ -34,7 +36,7 @@ class MirrorlistForm(forms.Form): @csrf_exempt def generate_mirrorlist(request): - if request.REQUEST.get('country', ''): + if request.method == 'POST' or len(request.GET) > 0: form = MirrorlistForm(data=request.REQUEST) if form.is_valid(): countries = form.cleaned_data['country'] @@ -129,4 +131,37 @@ def status(request): }) return direct_to_template(request, 'mirrors/status.html', context) +class MirrorStatusJSONEncoder(DjangoJSONEncoder): + '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl + serialization. The base class takes care of datetime.datetime types.''' + url_attributes = ['url', 'protocol', 'last_sync', 'completion_pct', + 'delay', 'duration_avg', 'duration_stddev', 'score'] + + def default(self, obj): + if isinstance(obj, datetime.timedelta): + # always returned as integer seconds + return obj.days * 24 * 3600 + obj.seconds + if hasattr(obj, '__iter__'): + # mainly for queryset serialization + return list(obj) + if isinstance(obj, MirrorUrl): + data = {} + for attr in self.url_attributes: + data[attr] = getattr(obj, attr) + # separate because it isn't on the URL directly + data['country'] = obj.mirror.country + return data + if isinstance(obj, MirrorProtocol): + return unicode(obj) + return super(MirrorStatusJSONEncoder, self).default(obj) + +def status_json(request): + status_info = get_mirror_statuses() + data = status_info.copy() + data['version'] = 1 + to_json = simplejson.dumps(data, ensure_ascii=False, + cls=MirrorStatusJSONEncoder) + response = HttpResponse(to_json, mimetype='application/json') + return response + # vim: set ts=4 sw=4 et: |