From be3080f912e7379ae9f6b6e78126a941c2534fe4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 7 Dec 2010 11:55:08 -0600 Subject: Don't require country on mirrorlist submission Signed-off-by: Dan McGee --- mirrors/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/views.py b/mirrors/views.py index fb7d3361..b9df6ba1 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -34,7 +34,7 @@ class MirrorlistForm(forms.Form): @csrf_exempt def generate_mirrorlist(request): - if request.REQUEST.get('country', ''): + if request.method == 'POST' or len(request.GET) > 0: form = MirrorlistForm(data=request.REQUEST) if form.is_valid(): countries = form.cleaned_data['country'] -- cgit v1.2.3-2-g168b From 5db6522b35dacf974c0e3a6cfb9d4beebfcee520 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 7 Dec 2010 17:05:44 -0600 Subject: Fix mirror check unicode function Needed a few self. qualifiers in there. Signed-off-by: Dan McGee --- mirrors/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/models.py b/mirrors/models.py index f1286026..e070b1cd 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -80,7 +80,7 @@ class MirrorLog(models.Model): error = models.CharField(max_length=255, blank=True, default='') def __unicode__(self): - return "Check of %s at %s" % (url.url, check_time) + return "Check of %s at %s" % (self.url.url, self.check_time) class Meta: verbose_name = 'Mirror Check Log' -- cgit v1.2.3-2-g168b From 1553e2b5c93f1a8566af4ed07630910fce6bc3b1 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 12 Dec 2010 01:13:25 -0600 Subject: Use check count for this URL, not max of all mirrors Prevents a recently enabled mirror from getting unfairly represented as far as completion percentage goes. Signed-off-by: Dan McGee --- mirrors/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/utils.py b/mirrors/utils.py index c64d53c5..124b66e6 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -51,7 +51,7 @@ def get_mirror_statuses(cutoff=default_cutoff): check_frequency = None for url in urls: - url.completion_pct = float(url.success_count) / num_checks + url.completion_pct = float(url.success_count) / url.check_count if url.id in delays: url_delays = delays[url.id] d = sum(url_delays, datetime.timedelta()) / len(url_delays) -- cgit v1.2.3-2-g168b From 7a320edae13ccf7db48f44137a5d20a1bffe90a5 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 13 Dec 2010 13:40:01 -0600 Subject: Move database interaction out of the threaded section We were seeing a lot of hangs and long-running never-ending processes. This might be due to some multithreading issues within Django, so move the save() calls to a loop after the join() on the threads doing the mirror polling. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 36 ++++++++++++++++++------------ 1 file changed, 22 insertions(+), 14 deletions(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index bab79219..d7aad4e2 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -10,7 +10,6 @@ Usage: ./manage.py mirrorcheck """ from django.core.management.base import NoArgsCommand -from django.db.models import Q from datetime import datetime, timedelta import logging @@ -51,10 +50,10 @@ class Command(NoArgsCommand): return check_current_mirrors() -def parse_rfc3339_datetime(time): +def parse_rfc3339_datetime(time_string): # '2010-09-02 11:05:06+02:00' m = re.match('^(\d{4})-(\d{2})-(\d{2}) ' - '(\d{2}):(\d{2}):(\d{2})([-+])(\d{2}):(\d{2})', time) + '(\d{2}):(\d{2}):(\d{2})([-+])(\d{2}):(\d{2})', time_string) if m: vals = m.groups() parsed = datetime(int(vals[0]), int(vals[1]), int(vals[2]), @@ -123,46 +122,55 @@ def check_mirror_url(mirror_url): log.error = "Connection timed out." logger.debug("failed: %s, %s" % (url, log.error)) - log.save() return log -def mirror_url_worker(queue): +def mirror_url_worker(work, output): while True: try: - item = queue.get(block=False) + item = work.get(block=False) try: - check_mirror_url(item) + log = check_mirror_url(item) + output.put(log) finally: - queue.task_done() + work.task_done() except Empty: return 0 class MirrorCheckPool(object): def __init__(self, work, num_threads=10): self.tasks = Queue() - for i in work: + self.logs = Queue() + for i in list(work): self.tasks.put(i) self.threads = [] for i in range(num_threads): - thread = Thread(target=mirror_url_worker, args=(self.tasks,)) + thread = Thread(target=mirror_url_worker, + args=(self.tasks, self.logs)) thread.daemon = True self.threads.append(thread) - def run_and_join(self): + def run(self): logger.debug("starting threads") for t in self.threads: t.start() logger.debug("joining on all threads") self.tasks.join() + logger.debug("processing log entries") + try: + while True: + log = self.logs.get(block=False) + log.save() + self.logs.task_done() + except Empty: + logger.debug("all log items saved to database") def check_current_mirrors(): urls = MirrorUrl.objects.filter( - Q(protocol__protocol__iexact='HTTP') | - Q(protocol__protocol__iexact='FTP'), + protocol__is_download=True, mirror__active=True, mirror__public=True) pool = MirrorCheckPool(urls) - pool.run_and_join() + pool.run() return 0 # For lack of a better place to put it, here is a query to get latest check -- cgit v1.2.3-2-g168b From 7b969772cc9f280255ef51d55f1e1ed9123af1fd Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 13 Dec 2010 13:58:40 -0600 Subject: Add a mirror status JSON view Requested in FS#21144. This should provide most if not all of the data that was provided on the archlinux.de website, although there are some differences in what is returned to the user. It is nearly the same data as that provided in the HTML view, the difference being things are a bit more machine-friendly and the list is not split into good and bad portions. Signed-off-by: Dan McGee --- mirrors/views.py | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) (limited to 'mirrors') diff --git a/mirrors/views.py b/mirrors/views.py index b9df6ba1..a2b94de8 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -1,10 +1,12 @@ from django import forms +from django.core.serializers.json import DjangoJSONEncoder from django.db.models import Avg, Count, Max, Min, StdDev from django.db.models import Q -from django.http import Http404 +from django.http import Http404, HttpResponse from django.shortcuts import get_object_or_404 from django.views.decorators.csrf import csrf_exempt from django.views.generic.simple import direct_to_template +from django.utils import simplejson from main.utils import make_choice from .models import Mirror, MirrorUrl, MirrorProtocol @@ -129,4 +131,37 @@ def status(request): }) return direct_to_template(request, 'mirrors/status.html', context) +class MirrorStatusJSONEncoder(DjangoJSONEncoder): + '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl + serialization. The base class takes care of datetime.datetime types.''' + url_attributes = ['url', 'protocol', 'last_sync', 'completion_pct', + 'delay', 'duration_avg', 'duration_stddev', 'score'] + + def default(self, obj): + if isinstance(obj, datetime.timedelta): + # always returned as integer seconds + return obj.days * 24 * 3600 + obj.seconds + if hasattr(obj, '__iter__'): + # mainly for queryset serialization + return list(obj) + if isinstance(obj, MirrorUrl): + data = {} + for attr in self.url_attributes: + data[attr] = getattr(obj, attr) + # separate because it isn't on the URL directly + data['country'] = obj.mirror.country + return data + if isinstance(obj, MirrorProtocol): + return unicode(obj) + return super(MirrorStatusJSONEncoder, self).default(obj) + +def status_json(request): + status_info = get_mirror_statuses() + data = status_info.copy() + data['version'] = 1 + to_json = simplejson.dumps(data, ensure_ascii=False, + cls=MirrorStatusJSONEncoder) + response = HttpResponse(to_json, mimetype='application/json') + return response + # vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b