diff options
Diffstat (limited to 'mirrors')
| -rw-r--r-- | mirrors/admin.py | 20 | ||||
| -rw-r--r-- | mirrors/management/commands/mirrorcheck.py | 17 | ||||
| -rw-r--r-- | mirrors/management/commands/mirrorresolv.py | 11 | ||||
| -rw-r--r-- | mirrors/models.py | 40 | ||||
| -rw-r--r-- | mirrors/urls.py | 10 | ||||
| -rw-r--r-- | mirrors/urls_mirrorlist.py | 12 |
6 files changed, 87 insertions, 23 deletions
diff --git a/mirrors/admin.py b/mirrors/admin.py index 6990cca2..b9c2876a 100644 --- a/mirrors/admin.py +++ b/mirrors/admin.py @@ -1,4 +1,5 @@ import re +from urlparse import urlparse, urlunsplit from django import forms from django.contrib import admin @@ -9,15 +10,26 @@ class MirrorUrlForm(forms.ModelForm): class Meta: model = MirrorUrl def clean_url(self): + # is this a valid-looking URL? + url_parts = urlparse(self.cleaned_data["url"]) + if not url_parts.scheme: + raise forms.ValidationError("No URL scheme (protocol) provided.") + if not url_parts.netloc: + raise forms.ValidationError("No URL host provided.") + if url_parts.params or url_parts.query or url_parts.fragment: + raise forms.ValidationError( + "URL parameters, query, and fragment elements are not supported.") # ensure we always save the URL with a trailing slash - url = self.cleaned_data["url"].strip() - if url[-1] == '/': - return url - return url + '/' + path = url_parts.path + if not path.endswith('/'): + path += '/' + url = urlunsplit((url_parts.scheme, url_parts.netloc, path, '', '')) + return url class MirrorUrlInlineAdmin(admin.TabularInline): model = MirrorUrl form = MirrorUrlForm + readonly_fields = ('protocol', 'has_ipv4', 'has_ipv6') extra = 3 # ripped off from django.forms.fields, adding netmask ability diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 44d2b364..51be71ea 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -10,7 +10,9 @@ Usage: ./manage.py mirrorcheck """ from django.core.management.base import NoArgsCommand +from django.db import transaction +from collections import deque from datetime import datetime, timedelta import logging import re @@ -130,7 +132,7 @@ def mirror_url_worker(work, output): item = work.get(block=False) try: log = check_mirror_url(item) - output.put(log) + output.append(log) finally: work.task_done() except Empty: @@ -139,7 +141,7 @@ def mirror_url_worker(work, output): class MirrorCheckPool(object): def __init__(self, work, num_threads=10): self.tasks = Queue() - self.logs = Queue() + self.logs = deque() for i in list(work): self.tasks.put(i) self.threads = [] @@ -149,6 +151,7 @@ class MirrorCheckPool(object): thread.daemon = True self.threads.append(thread) + @transaction.commit_on_success def run(self): logger.debug("starting threads") for t in self.threads: @@ -156,13 +159,9 @@ class MirrorCheckPool(object): logger.debug("joining on all threads") self.tasks.join() logger.debug("processing log entries") - try: - while True: - log = self.logs.get(block=False) - log.save() - self.logs.task_done() - except Empty: - logger.debug("all log items saved to database") + for log in self.logs: + log.save() + logger.debug("log entries saved") def check_current_mirrors(): urls = MirrorUrl.objects.filter( diff --git a/mirrors/management/commands/mirrorresolv.py b/mirrors/management/commands/mirrorresolv.py index 8cbd51a3..8a628bd4 100644 --- a/mirrors/management/commands/mirrorresolv.py +++ b/mirrors/management/commands/mirrorresolv.py @@ -12,7 +12,6 @@ from django.core.management.base import NoArgsCommand import sys import logging -from urlparse import urlparse import socket from mirrors.models import MirrorUrl @@ -42,15 +41,13 @@ def resolve_mirrors(): logger.debug("requesting list of mirror URLs") for mirrorurl in MirrorUrl.objects.filter(mirror__active=True): try: - hostname = urlparse(mirrorurl.url).hostname - logger.debug("resolving %3i (%s)", mirrorurl.id, hostname) - info = socket.getaddrinfo(hostname, None, 0, socket.SOCK_STREAM) - families = [x[0] for x in info] + logger.debug("resolving %3i (%s)", mirrorurl.id, mirrorurl.hostname) + families = mirrorurl.address_families() mirrorurl.has_ipv4 = socket.AF_INET in families mirrorurl.has_ipv6 = socket.AF_INET6 in families - logger.debug("%s: v4: %s v6: %s", hostname, + logger.debug("%s: v4: %s v6: %s", mirrorurl.hostname, mirrorurl.has_ipv4, mirrorurl.has_ipv6) - mirrorurl.save() + mirrorurl.save(force_update=True) except socket.error, e: logger.warn("error resolving %s: %s", hostname, e) diff --git a/mirrors/models.py b/mirrors/models.py index e070b1cd..401821a8 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -1,4 +1,8 @@ from django.db import models +from django.core.exceptions import ValidationError + +import socket +from urlparse import urlparse TIER_CHOICES = ( (0, 'Tier 0'), @@ -47,13 +51,43 @@ class MirrorProtocol(models.Model): class Meta: verbose_name = 'Mirror Protocol' + ordering = ('protocol',) class MirrorUrl(models.Model): url = models.CharField(max_length=255) - protocol = models.ForeignKey(MirrorProtocol, related_name="urls") + protocol = models.ForeignKey(MirrorProtocol, related_name="urls", + editable=False) mirror = models.ForeignKey(Mirror, related_name="urls") - has_ipv4 = models.BooleanField("IPv4 capable", default=True) - has_ipv6 = models.BooleanField("IPv6 capable", default=False) + has_ipv4 = models.BooleanField("IPv4 capable", default=True, + editable=False) + has_ipv6 = models.BooleanField("IPv6 capable", default=False, + editable=False) + + def address_families(self): + hostname = urlparse(self.url).hostname + info = socket.getaddrinfo(hostname, None, 0, socket.SOCK_STREAM) + families = [x[0] for x in info] + return families + + @property + def hostname(self): + return urlparse(self.url).hostname + + def clean(self): + try: + # Auto-map the protocol field by looking at the URL + protocol = urlparse(self.url).scheme + self.protocol = MirrorProtocol.objects.get(protocol=protocol) + except Exception as e: + raise ValidationError(e) + try: + families = self.address_families() + self.has_ipv4 = socket.AF_INET in families + self.has_ipv6 = socket.AF_INET6 in families + except socket.error as e: + # We don't fail in this case; we'll just set both to False + self.has_ipv4 = False + self.has_ipv6 = False def __unicode__(self): return self.url diff --git a/mirrors/urls.py b/mirrors/urls.py new file mode 100644 index 00000000..fed9c807 --- /dev/null +++ b/mirrors/urls.py @@ -0,0 +1,10 @@ +from django.conf.urls.defaults import patterns + +urlpatterns = patterns('mirrors.views', + (r'^$', 'mirrors', {}, 'mirror-list'), + (r'^status/$', 'status', {}, 'mirror-status'), + (r'^status/json/$', 'status_json', {}, 'mirror-status-json'), + (r'^(?P<name>[\.\-\w]+)/$', 'mirror_details'), +) + +# vim: set ts=4 sw=4 et: diff --git a/mirrors/urls_mirrorlist.py b/mirrors/urls_mirrorlist.py new file mode 100644 index 00000000..70bc18d2 --- /dev/null +++ b/mirrors/urls_mirrorlist.py @@ -0,0 +1,12 @@ +from django.conf.urls.defaults import patterns + +urlpatterns = patterns('mirrors.views', + (r'^$', 'generate_mirrorlist', {}, 'mirrorlist'), + (r'^all/$', 'find_mirrors', {'countries': ['all']}), + (r'^all/ftp/$', 'find_mirrors', + {'countries': ['all'], 'protocols': ['ftp']}), + (r'^all/http/$', 'find_mirrors', + {'countries': ['all'], 'protocols': ['http']}), +) + +# vim: set ts=4 sw=4 et: |