From b9fdcd06222c674d5fabcf5a4ab6bc55f268c757 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 22 Feb 2011 15:29:08 -0600 Subject: Modularize URLs Make some additional URL config files that can be included so we aren't trying to do so much in the top level config. This also allows us to branch a bit more rather than go linear down the rather lengthy list. Signed-off-by: Dan McGee --- mirrors/urls.py | 10 ++++++++++ mirrors/urls_mirrorlist.py | 12 ++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 mirrors/urls.py create mode 100644 mirrors/urls_mirrorlist.py (limited to 'mirrors') diff --git a/mirrors/urls.py b/mirrors/urls.py new file mode 100644 index 00000000..fed9c807 --- /dev/null +++ b/mirrors/urls.py @@ -0,0 +1,10 @@ +from django.conf.urls.defaults import patterns + +urlpatterns = patterns('mirrors.views', + (r'^$', 'mirrors', {}, 'mirror-list'), + (r'^status/$', 'status', {}, 'mirror-status'), + (r'^status/json/$', 'status_json', {}, 'mirror-status-json'), + (r'^(?P[\.\-\w]+)/$', 'mirror_details'), +) + +# vim: set ts=4 sw=4 et: diff --git a/mirrors/urls_mirrorlist.py b/mirrors/urls_mirrorlist.py new file mode 100644 index 00000000..70bc18d2 --- /dev/null +++ b/mirrors/urls_mirrorlist.py @@ -0,0 +1,12 @@ +from django.conf.urls.defaults import patterns + +urlpatterns = patterns('mirrors.views', + (r'^$', 'generate_mirrorlist', {}, 'mirrorlist'), + (r'^all/$', 'find_mirrors', {'countries': ['all']}), + (r'^all/ftp/$', 'find_mirrors', + {'countries': ['all'], 'protocols': ['ftp']}), + (r'^all/http/$', 'find_mirrors', + {'countries': ['all'], 'protocols': ['http']}), +) + +# vim: set ts=4 sw=4 et: -- cgit v1.2.3-2-g168b From fc7fc1d10a13e9b21d222435ba12affd1d08822f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 16 Feb 2011 21:51:23 -0600 Subject: Slight tweaks to mirror commands We know we are doing updates when setting IPv4/IPv6 information, so set force_update to True to save the useless select query on each save(). For mirror checks, use a less cumbersome deque for the results since it is also thread-safe, and have all the log entries committed in one go. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 17 ++++++++--------- mirrors/management/commands/mirrorresolv.py | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) (limited to 'mirrors') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 44d2b364..51be71ea 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -10,7 +10,9 @@ Usage: ./manage.py mirrorcheck """ from django.core.management.base import NoArgsCommand +from django.db import transaction +from collections import deque from datetime import datetime, timedelta import logging import re @@ -130,7 +132,7 @@ def mirror_url_worker(work, output): item = work.get(block=False) try: log = check_mirror_url(item) - output.put(log) + output.append(log) finally: work.task_done() except Empty: @@ -139,7 +141,7 @@ def mirror_url_worker(work, output): class MirrorCheckPool(object): def __init__(self, work, num_threads=10): self.tasks = Queue() - self.logs = Queue() + self.logs = deque() for i in list(work): self.tasks.put(i) self.threads = [] @@ -149,6 +151,7 @@ class MirrorCheckPool(object): thread.daemon = True self.threads.append(thread) + @transaction.commit_on_success def run(self): logger.debug("starting threads") for t in self.threads: @@ -156,13 +159,9 @@ class MirrorCheckPool(object): logger.debug("joining on all threads") self.tasks.join() logger.debug("processing log entries") - try: - while True: - log = self.logs.get(block=False) - log.save() - self.logs.task_done() - except Empty: - logger.debug("all log items saved to database") + for log in self.logs: + log.save() + logger.debug("log entries saved") def check_current_mirrors(): urls = MirrorUrl.objects.filter( diff --git a/mirrors/management/commands/mirrorresolv.py b/mirrors/management/commands/mirrorresolv.py index 8cbd51a3..77e7a223 100644 --- a/mirrors/management/commands/mirrorresolv.py +++ b/mirrors/management/commands/mirrorresolv.py @@ -50,7 +50,7 @@ def resolve_mirrors(): mirrorurl.has_ipv6 = socket.AF_INET6 in families logger.debug("%s: v4: %s v6: %s", hostname, mirrorurl.has_ipv4, mirrorurl.has_ipv6) - mirrorurl.save() + mirrorurl.save(force_update=True) except socket.error, e: logger.warn("error resolving %s: %s", hostname, e) -- cgit v1.2.3-2-g168b From db1524fd64e8b5c0f43cfed7643034ee764f55fd Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 27 Feb 2011 12:31:06 -0600 Subject: Auto map the protocol URL field And perform better validation when doing so. Signed-off-by: Dan McGee --- mirrors/admin.py | 20 ++++++++++++++++---- mirrors/models.py | 14 +++++++++++++- 2 files changed, 29 insertions(+), 5 deletions(-) (limited to 'mirrors') diff --git a/mirrors/admin.py b/mirrors/admin.py index 6990cca2..f6a72cf0 100644 --- a/mirrors/admin.py +++ b/mirrors/admin.py @@ -1,4 +1,5 @@ import re +from urlparse import urlparse, urlunsplit from django import forms from django.contrib import admin @@ -9,15 +10,26 @@ class MirrorUrlForm(forms.ModelForm): class Meta: model = MirrorUrl def clean_url(self): + # is this a valid-looking URL? + url_parts = urlparse(self.cleaned_data["url"]) + if not url_parts.scheme: + raise forms.ValidationError("No URL scheme (protocol) provided.") + if not url_parts.netloc: + raise forms.ValidationError("No URL host provided.") + if url_parts.params or url_parts.query or url_parts.fragment: + raise forms.ValidationError( + "URL parameters, query, and fragment elements are not supported.") # ensure we always save the URL with a trailing slash - url = self.cleaned_data["url"].strip() - if url[-1] == '/': - return url - return url + '/' + path = url_parts.path + if not path.endswith('/'): + path += '/' + url = urlunsplit((url_parts.scheme, url_parts.netloc, path, '', '')) + return url class MirrorUrlInlineAdmin(admin.TabularInline): model = MirrorUrl form = MirrorUrlForm + readonly_fields = ('protocol',) extra = 3 # ripped off from django.forms.fields, adding netmask ability diff --git a/mirrors/models.py b/mirrors/models.py index e070b1cd..b763fb63 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -1,4 +1,7 @@ from django.db import models +from django.core.exceptions import ValidationError + +from urlparse import urlparse TIER_CHOICES = ( (0, 'Tier 0'), @@ -50,11 +53,20 @@ class MirrorProtocol(models.Model): class MirrorUrl(models.Model): url = models.CharField(max_length=255) - protocol = models.ForeignKey(MirrorProtocol, related_name="urls") + protocol = models.ForeignKey(MirrorProtocol, related_name="urls", + editable=False) mirror = models.ForeignKey(Mirror, related_name="urls") has_ipv4 = models.BooleanField("IPv4 capable", default=True) has_ipv6 = models.BooleanField("IPv6 capable", default=False) + def clean(self): + try: + # Auto-map the protocol field by looking at the URL + protocol = urlparse(self.url).scheme + self.protocol = MirrorProtocol.objects.get(protocol=protocol) + except Exception as e: + raise ValidationError(e) + def __unicode__(self): return self.url -- cgit v1.2.3-2-g168b From 6d5909ca5dc8934cacd969e964573989dc6b0417 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 27 Feb 2011 12:44:30 -0600 Subject: Auto-resolve mirror URLs on save This prevents people from having to mess with these checkboxes at all in the admin, and we incur no delay on their initial values being correct waiting for the cron job to run. Signed-off-by: Dan McGee --- mirrors/admin.py | 2 +- mirrors/management/commands/mirrorresolv.py | 9 +++------ mirrors/models.py | 25 +++++++++++++++++++++++-- 3 files changed, 27 insertions(+), 9 deletions(-) (limited to 'mirrors') diff --git a/mirrors/admin.py b/mirrors/admin.py index f6a72cf0..b9c2876a 100644 --- a/mirrors/admin.py +++ b/mirrors/admin.py @@ -29,7 +29,7 @@ class MirrorUrlForm(forms.ModelForm): class MirrorUrlInlineAdmin(admin.TabularInline): model = MirrorUrl form = MirrorUrlForm - readonly_fields = ('protocol',) + readonly_fields = ('protocol', 'has_ipv4', 'has_ipv6') extra = 3 # ripped off from django.forms.fields, adding netmask ability diff --git a/mirrors/management/commands/mirrorresolv.py b/mirrors/management/commands/mirrorresolv.py index 77e7a223..8a628bd4 100644 --- a/mirrors/management/commands/mirrorresolv.py +++ b/mirrors/management/commands/mirrorresolv.py @@ -12,7 +12,6 @@ from django.core.management.base import NoArgsCommand import sys import logging -from urlparse import urlparse import socket from mirrors.models import MirrorUrl @@ -42,13 +41,11 @@ def resolve_mirrors(): logger.debug("requesting list of mirror URLs") for mirrorurl in MirrorUrl.objects.filter(mirror__active=True): try: - hostname = urlparse(mirrorurl.url).hostname - logger.debug("resolving %3i (%s)", mirrorurl.id, hostname) - info = socket.getaddrinfo(hostname, None, 0, socket.SOCK_STREAM) - families = [x[0] for x in info] + logger.debug("resolving %3i (%s)", mirrorurl.id, mirrorurl.hostname) + families = mirrorurl.address_families() mirrorurl.has_ipv4 = socket.AF_INET in families mirrorurl.has_ipv6 = socket.AF_INET6 in families - logger.debug("%s: v4: %s v6: %s", hostname, + logger.debug("%s: v4: %s v6: %s", mirrorurl.hostname, mirrorurl.has_ipv4, mirrorurl.has_ipv6) mirrorurl.save(force_update=True) except socket.error, e: diff --git a/mirrors/models.py b/mirrors/models.py index b763fb63..7c269898 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -1,6 +1,7 @@ from django.db import models from django.core.exceptions import ValidationError +import socket from urlparse import urlparse TIER_CHOICES = ( @@ -56,8 +57,20 @@ class MirrorUrl(models.Model): protocol = models.ForeignKey(MirrorProtocol, related_name="urls", editable=False) mirror = models.ForeignKey(Mirror, related_name="urls") - has_ipv4 = models.BooleanField("IPv4 capable", default=True) - has_ipv6 = models.BooleanField("IPv6 capable", default=False) + has_ipv4 = models.BooleanField("IPv4 capable", default=True, + editable=False) + has_ipv6 = models.BooleanField("IPv6 capable", default=False, + editable=False) + + def address_families(self): + hostname = urlparse(self.url).hostname + info = socket.getaddrinfo(hostname, None, 0, socket.SOCK_STREAM) + families = [x[0] for x in info] + return families + + @property + def hostname(self): + return urlparse(self.url).hostname def clean(self): try: @@ -66,6 +79,14 @@ class MirrorUrl(models.Model): self.protocol = MirrorProtocol.objects.get(protocol=protocol) except Exception as e: raise ValidationError(e) + try: + families = self.address_families() + self.has_ipv4 = socket.AF_INET in families + self.has_ipv6 = socket.AF_INET6 in families + except socket.error as e: + # We don't fail in this case; we'll just set both to False + self.has_ipv4 = False + self.has_ipv6 = False def __unicode__(self): return self.url -- cgit v1.2.3-2-g168b From f29700ce67ad323e8c68dacd776cf3e991721104 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 3 Mar 2011 15:00:32 -0600 Subject: Add ordering to mirror protocols Signed-off-by: Dan McGee --- mirrors/models.py | 1 + 1 file changed, 1 insertion(+) (limited to 'mirrors') diff --git a/mirrors/models.py b/mirrors/models.py index 7c269898..401821a8 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -51,6 +51,7 @@ class MirrorProtocol(models.Model): class Meta: verbose_name = 'Mirror Protocol' + ordering = ('protocol',) class MirrorUrl(models.Model): url = models.CharField(max_length=255) -- cgit v1.2.3-2-g168b