summaryrefslogtreecommitdiff
path: root/mirrors
diff options
context:
space:
mode:
authorLuke Shumaker <LukeShu@sbcglobal.net>2013-04-22 00:36:57 -0400
committerLuke Shumaker <LukeShu@sbcglobal.net>2013-04-22 00:36:57 -0400
commitdf7a6aa620af6a165bdacd755757f8cb1179331c (patch)
tree384b4c62d1f50d8effb733d81d2a810666807624 /mirrors
parent94f972bb892dbf9a86f089f1872ae6d849c0cd0e (diff)
parenta22557811a24b68ef85d4271787c48d8d1e4fc99 (diff)
Merge branch 'archweb-generic2'
Conflicts: README.BRANDING local_settings.py.example packages/templatetags/package_extras.py public/views.py releng/views.py settings.py sitestatic/archnavbar/archnavbar.css sitestatic/silhouette.png templates/base.html templates/packages/differences.html templates/packages/opensearch.xml templates/packages/search.html templates/public/donate.html templates/public/download.html templates/public/feeds.html templates/public/index.html urls.py
Diffstat (limited to 'mirrors')
-rw-r--r--mirrors/admin.py36
-rw-r--r--mirrors/fields.py49
-rw-r--r--mirrors/fixtures/mirrorprotocols.json13
-rw-r--r--mirrors/management/commands/mirrorcheck.py200
-rw-r--r--mirrors/management/commands/mirrorresolv.py8
-rw-r--r--mirrors/migrations/0004_auto__add_field_mirrorprotocol_is_download.py2
-rw-r--r--mirrors/migrations/0006_auto__add_field_mirrorurl_has_ipv4__add_field_mirrorurl_has_ipv6.py4
-rw-r--r--mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py2
-rw-r--r--mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py66
-rw-r--r--mirrors/migrations/0018_auto__add_field_mirror_alternate_email.py68
-rw-r--r--mirrors/migrations/0019_move_country_data_to_url.py74
-rw-r--r--mirrors/migrations/0020_auto__del_field_mirror_country.py70
-rw-r--r--mirrors/migrations/0021_auto__chg_field_mirrorrsync_ip.py66
-rw-r--r--mirrors/migrations/0022_auto__add_checklocation.py83
-rw-r--r--mirrors/migrations/0023_auto__add_field_mirrorurl_created__add_field_mirrorrsync_created__add_.py97
-rw-r--r--mirrors/migrations/0024_auto__add_field_mirrorlog_location.py83
-rw-r--r--mirrors/migrations/0025_auto__chg_field_mirrorrsync_ip.py85
-rw-r--r--mirrors/models.py86
-rw-r--r--mirrors/static/mirror_status.js173
-rw-r--r--mirrors/urls.py4
-rw-r--r--mirrors/urls_mirrorlist.py5
-rw-r--r--mirrors/utils.py179
-rw-r--r--mirrors/views.py239
23 files changed, 1475 insertions, 217 deletions
diff --git a/mirrors/admin.py b/mirrors/admin.py
index b7b9894c..9c88207d 100644
--- a/mirrors/admin.py
+++ b/mirrors/admin.py
@@ -1,10 +1,11 @@
-import re
from urlparse import urlparse, urlunsplit
from django import forms
from django.contrib import admin
-from .models import Mirror, MirrorProtocol, MirrorUrl, MirrorRsync
+from .models import (Mirror, MirrorProtocol, MirrorUrl, MirrorRsync,
+ CheckLocation)
+
class MirrorUrlForm(forms.ModelForm):
class Meta:
@@ -26,33 +27,25 @@ class MirrorUrlForm(forms.ModelForm):
url = urlunsplit((url_parts.scheme, url_parts.netloc, path, '', ''))
return url
+
class MirrorUrlInlineAdmin(admin.TabularInline):
model = MirrorUrl
form = MirrorUrlForm
readonly_fields = ('protocol', 'has_ipv4', 'has_ipv6')
extra = 3
-# ripped off from django.forms.fields, adding netmask ability
-IPV4NM_RE = re.compile(r'^(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}(/(\d|[1-2]\d|3[0-2])){0,1}$')
-
-class IPAddressNetmaskField(forms.fields.RegexField):
- default_error_messages = {
- 'invalid': u'Enter a valid IPv4 address, possibly including netmask.',
- }
-
- def __init__(self, *args, **kwargs):
- super(IPAddressNetmaskField, self).__init__(IPV4NM_RE, *args, **kwargs)
class MirrorRsyncForm(forms.ModelForm):
class Meta:
model = MirrorRsync
- ip = IPAddressNetmaskField(label='IP')
+
class MirrorRsyncInlineAdmin(admin.TabularInline):
model = MirrorRsync
form = MirrorRsyncForm
extra = 2
+
class MirrorAdminForm(forms.ModelForm):
class Meta:
model = Mirror
@@ -60,22 +53,31 @@ class MirrorAdminForm(forms.ModelForm):
queryset=Mirror.objects.filter(tier__gte=0, tier__lte=1),
required=False)
+
class MirrorAdmin(admin.ModelAdmin):
form = MirrorAdminForm
- list_display = ('name', 'tier', 'country', 'active', 'public',
- 'isos', 'admin_email')
- list_filter = ('tier', 'active', 'public', 'country')
- search_fields = ('name',)
+ list_display = ('name', 'tier', 'active', 'public',
+ 'isos', 'admin_email', 'alternate_email')
+ list_filter = ('tier', 'active', 'public')
+ search_fields = ('name', 'admin_email', 'alternate_email')
inlines = [
MirrorUrlInlineAdmin,
MirrorRsyncInlineAdmin,
]
+
class MirrorProtocolAdmin(admin.ModelAdmin):
list_display = ('protocol', 'is_download', 'default')
list_filter = ('is_download', 'default')
+
+class CheckLocationAdmin(admin.ModelAdmin):
+ list_display = ('hostname', 'source_ip', 'country', 'created')
+ search_fields = ('hostname', 'source_ip')
+
+
admin.site.register(Mirror, MirrorAdmin)
admin.site.register(MirrorProtocol, MirrorProtocolAdmin)
+admin.site.register(CheckLocation, CheckLocationAdmin)
# vim: set ts=4 sw=4 et:
diff --git a/mirrors/fields.py b/mirrors/fields.py
new file mode 100644
index 00000000..206c9d7d
--- /dev/null
+++ b/mirrors/fields.py
@@ -0,0 +1,49 @@
+from IPy import IP
+
+from django import forms
+from django.core import validators
+from django.core.exceptions import ValidationError
+from django.db import models
+from south.modelsinspector import add_introspection_rules
+
+
+class IPNetworkFormField(forms.Field):
+ def to_python(self, value):
+ if value in validators.EMPTY_VALUES:
+ return None
+ try:
+ value = IP(value)
+ except ValueError as e:
+ raise ValidationError(str(e))
+ return value
+
+
+class IPNetworkField(models.Field):
+ __metaclass__ = models.SubfieldBase
+ description = "IPv4 or IPv6 address or subnet"
+
+ def __init__(self, *args, **kwargs):
+ kwargs['max_length'] = 44
+ super(IPNetworkField, self).__init__(*args, **kwargs)
+
+ def get_internal_type(self):
+ return "IPAddressField"
+
+ def to_python(self, value):
+ if not value:
+ return None
+ return IP(value)
+
+ def get_prep_value(self, value):
+ value = self.to_python(value)
+ if not value:
+ return None
+ return str(value)
+
+ def formfield(self, **kwargs):
+ defaults = {'form_class': IPNetworkFormField}
+ defaults.update(kwargs)
+ return super(IPNetworkField, self).formfield(**defaults)
+
+
+add_introspection_rules([], ["^mirrors\.fields\.IPNetworkField"])
diff --git a/mirrors/fixtures/mirrorprotocols.json b/mirrors/fixtures/mirrorprotocols.json
index 72ed1a7f..8822ef8e 100644
--- a/mirrors/fixtures/mirrorprotocols.json
+++ b/mirrors/fixtures/mirrorprotocols.json
@@ -7,7 +7,7 @@
"default": true,
"protocol": "http"
}
- },
+ },
{
"pk": 2,
"model": "mirrors.mirrorprotocol",
@@ -16,7 +16,7 @@
"default": false,
"protocol": "ftp"
}
- },
+ },
{
"pk": 3,
"model": "mirrors.mirrorprotocol",
@@ -25,5 +25,14 @@
"default": false,
"protocol": "rsync"
}
+ },
+ {
+ "pk": 5,
+ "model": "mirrors.mirrorprotocol",
+ "fields": {
+ "is_download": true,
+ "default": false,
+ "protocol": "https"
+ }
}
]
diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py
index 7ffb7773..e7dd7b49 100644
--- a/mirrors/management/commands/mirrorcheck.py
+++ b/mirrors/management/commands/mirrorcheck.py
@@ -9,24 +9,30 @@ we encounter errors, record those as well.
Usage: ./manage.py mirrorcheck
"""
-from django.core.management.base import NoArgsCommand
-from django.db import transaction
-
from collections import deque
from datetime import datetime
+from httplib import HTTPException
import logging
+import os
+from optparse import make_option
+from pytz import utc
import re
import socket
+import subprocess
import sys
import time
+import tempfile
from threading import Thread
import types
-from pytz import utc
from Queue import Queue, Empty
import urllib2
-from main.utils import utc_now
-from mirrors.models import MirrorUrl, MirrorLog
+from django.core.management.base import NoArgsCommand
+from django.db import transaction
+from django.utils.timezone import now
+
+from mirrors.models import MirrorUrl, MirrorLog, CheckLocation
+
logging.basicConfig(
level=logging.WARNING,
@@ -35,7 +41,14 @@ logging.basicConfig(
stream=sys.stderr)
logger = logging.getLogger()
+
class Command(NoArgsCommand):
+ option_list = NoArgsCommand.option_list + (
+ make_option('-t', '--timeout', dest='timeout', type='float', default=10.0,
+ help='Timeout value for connecting to URL'),
+ make_option('-l', '--location', dest='location', type='int',
+ help='ID of CheckLocation object to use for this run'),
+ )
help = "Runs a check on all known mirror URLs to determine their up-to-date status."
def handle_noargs(self, **options):
@@ -47,36 +60,67 @@ class Command(NoArgsCommand):
elif v == 2:
logger.level = logging.DEBUG
- return check_current_mirrors()
+ timeout = options.get('timeout')
+
+ urls = MirrorUrl.objects.select_related('protocol').filter(
+ mirror__active=True, mirror__public=True)
+
+ location = options.get('location', None)
+ if location:
+ location = CheckLocation.objects.get(id=location)
+ family = location.family
+ monkeypatch_getaddrinfo(family)
+ if family == socket.AF_INET6:
+ urls = urls.filter(has_ipv6=True)
+ elif family == socket.AF_INET:
+ urls = urls.filter(has_ipv4=True)
+
+ pool = MirrorCheckPool(urls, location, timeout)
+ pool.run()
+ return 0
+
+
+def monkeypatch_getaddrinfo(force_family=socket.AF_INET):
+ '''Force the Python socket module to connect over the designated family;
+ e.g. socket.AF_INET or socket.AF_INET6.'''
+ orig = socket.getaddrinfo
+
+ def wrapper(host, port, family=0, socktype=0, proto=0, flags=0):
+ return orig(host, port, force_family, socktype, proto, flags)
+
+ socket.getaddrinfo = wrapper
+
+
+def parse_lastsync(log, data):
+ '''lastsync file should be an epoch value created by us.'''
+ try:
+ parsed_time = datetime.utcfromtimestamp(int(data))
+ log.last_sync = parsed_time.replace(tzinfo=utc)
+ except ValueError:
+ # it is bad news to try logging the lastsync value;
+ # sometimes we get a crazy-encoded web page.
+ # if we couldn't parse a time, this is a failure.
+ log.last_sync = None
+ log.error = "Could not parse time from lastsync"
+ log.is_success = False
+
-def check_mirror_url(mirror_url):
+def check_mirror_url(mirror_url, location, timeout):
url = mirror_url.url + 'lastsync'
logger.info("checking URL %s", url)
- log = MirrorLog(url=mirror_url, check_time=utc_now())
+ log = MirrorLog(url=mirror_url, check_time=now(), location=location)
+ headers = {'User-Agent': 'archweb/1.0'}
+ req = urllib2.Request(url, None, headers)
+ start = time.time()
try:
- start = time.time()
- result = urllib2.urlopen(url, timeout=10)
+ result = urllib2.urlopen(req, timeout=timeout)
data = result.read()
result.close()
end = time.time()
- # lastsync should be an epoch value created by us
- parsed_time = None
- try:
- parsed_time = datetime.utcfromtimestamp(int(data))
- parsed_time = parsed_time.replace(tzinfo=utc)
- except ValueError:
- # it is bad news to try logging the lastsync value;
- # sometimes we get a crazy-encoded web page.
- pass
-
- log.last_sync = parsed_time
- # if we couldn't parse a time, this is a failure
- if parsed_time is None:
- log.error = "Could not parse time from lastsync"
- log.is_success = False
+ parse_lastsync(log, data)
log.duration = end - start
logger.debug("success: %s, %.2f", url, log.duration)
- except urllib2.HTTPError, e:
+ except urllib2.HTTPError as e:
if e.code == 404:
# we have a duration, just not a success
end = time.time()
@@ -84,7 +128,7 @@ def check_mirror_url(mirror_url):
log.is_success = False
log.error = str(e)
logger.debug("failed: %s, %s", url, log.error)
- except urllib2.URLError, e:
+ except urllib2.URLError as e:
log.is_success = False
log.error = e.reason
if isinstance(e.reason, types.StringTypes) and \
@@ -97,35 +141,102 @@ def check_mirror_url(mirror_url):
elif isinstance(e.reason, socket.error):
log.error = e.reason.args[1]
logger.debug("failed: %s, %s", url, log.error)
- except socket.timeout, e:
+ except HTTPException:
+ # e.g., BadStatusLine
+ log.is_success = False
+ log.error = "Exception in processing HTTP request."
+ logger.debug("failed: %s, %s", url, log.error)
+ except socket.timeout:
log.is_success = False
log.error = "Connection timed out."
logger.debug("failed: %s, %s", url, log.error)
+ except socket.error as e:
+ log.is_success = False
+ log.error = str(e)
+ logger.debug("failed: %s, %s", url, log.error)
+
+ return log
+
+
+def check_rsync_url(mirror_url, location, timeout):
+ url = mirror_url.url + 'lastsync'
+ logger.info("checking URL %s", url)
+ log = MirrorLog(url=mirror_url, check_time=now(), location=location)
+
+ tempdir = tempfile.mkdtemp()
+ ipopt = ''
+ if location:
+ if location.family == socket.AF_INET6:
+ ipopt = '--ipv6'
+ elif location.family == socket.AF_INET:
+ ipopt = '--ipv4'
+ lastsync_path = os.path.join(tempdir, 'lastsync')
+ rsync_cmd = ["rsync", "--quiet", "--contimeout=%d" % timeout,
+ "--timeout=%d" % timeout]
+ if ipopt:
+ rsync_cmd.append(ipopt)
+ rsync_cmd.append(url)
+ rsync_cmd.append(lastsync_path)
+ try:
+ with open(os.devnull, 'w') as devnull:
+ logger.debug("rsync cmd: %s", ' '.join(rsync_cmd))
+ proc = subprocess.Popen(rsync_cmd, stdout=devnull,
+ stderr=subprocess.PIPE)
+ start = time.time()
+ _, errdata = proc.communicate()
+ end = time.time()
+ log.duration = end - start
+ if proc.returncode != 0:
+ logger.debug("error: %s, %s", url, errdata)
+ log.is_success = False
+ log.error = errdata.strip()
+ # look at rsync error code- if we had a command error or timed out,
+ # don't record a duration as it is misleading
+ if proc.returncode in (1, 30, 35):
+ log.duration = None
+ else:
+ logger.debug("success: %s, %.2f", url, log.duration)
+ with open(lastsync_path, 'r') as lastsync:
+ parse_lastsync(log, lastsync.read())
+ finally:
+ if os.path.exists(lastsync_path):
+ os.unlink(lastsync_path)
+ os.rmdir(tempdir)
return log
-def mirror_url_worker(work, output):
+
+def mirror_url_worker(work, output, location, timeout):
while True:
try:
- item = work.get(block=False)
+ url = work.get(block=False)
try:
- log = check_mirror_url(item)
- output.append(log)
+ if url.protocol.protocol == 'rsync':
+ log = check_rsync_url(url, location, timeout)
+ elif (url.protocol.protocol == 'ftp' and location and
+ location.family == socket.AF_INET6):
+ # IPv6 + FTP don't work; skip checking completely
+ log = None
+ else:
+ log = check_mirror_url(url, location, timeout)
+ if log:
+ output.append(log)
finally:
work.task_done()
except Empty:
return 0
+
class MirrorCheckPool(object):
- def __init__(self, work, num_threads=10):
+ def __init__(self, urls, location, timeout=10, num_threads=10):
self.tasks = Queue()
self.logs = deque()
- for i in list(work):
- self.tasks.put(i)
+ for url in list(urls):
+ self.tasks.put(url)
self.threads = []
for i in range(num_threads):
thread = Thread(target=mirror_url_worker,
- args=(self.tasks, self.logs))
+ args=(self.tasks, self.logs, location, timeout))
thread.daemon = True
self.threads.append(thread)
@@ -136,21 +247,8 @@ class MirrorCheckPool(object):
thread.start()
logger.debug("joining on all threads")
self.tasks.join()
- logger.debug("processing log entries")
+ logger.debug("processing %d log entries", len(self.logs))
MirrorLog.objects.bulk_create(self.logs)
logger.debug("log entries saved")
-def check_current_mirrors():
- urls = MirrorUrl.objects.filter(
- protocol__is_download=True,
- mirror__active=True, mirror__public=True)
-
- pool = MirrorCheckPool(urls)
- pool.run()
- return 0
-
-# For lack of a better place to put it, here is a query to get latest check
-# result joined with mirror details:
-# SELECT mu.*, m.*, ml.* FROM mirrors_mirrorurl mu JOIN mirrors_mirror m ON mu.mirror_id = m.id JOIN mirrors_mirrorlog ml ON mu.id = ml.url_id LEFT JOIN mirrors_mirrorlog ml2 ON ml.url_id = ml2.url_id AND ml.id < ml2.id WHERE ml2.id IS NULL AND m.active = 1 AND m.public = 1;
-
# vim: set ts=4 sw=4 et:
diff --git a/mirrors/management/commands/mirrorresolv.py b/mirrors/management/commands/mirrorresolv.py
index 4e812f2d..a6c2523e 100644
--- a/mirrors/management/commands/mirrorresolv.py
+++ b/mirrors/management/commands/mirrorresolv.py
@@ -41,13 +41,19 @@ def resolve_mirrors():
logger.debug("requesting list of mirror URLs")
for mirrorurl in MirrorUrl.objects.filter(mirror__active=True):
try:
+ # save old values, we can skip no-op updates this way
+ oldvals = (mirrorurl.has_ipv4, mirrorurl.has_ipv6)
logger.debug("resolving %3i (%s)", mirrorurl.id, mirrorurl.hostname)
families = mirrorurl.address_families()
mirrorurl.has_ipv4 = socket.AF_INET in families
mirrorurl.has_ipv6 = socket.AF_INET6 in families
logger.debug("%s: v4: %s v6: %s", mirrorurl.hostname,
mirrorurl.has_ipv4, mirrorurl.has_ipv6)
- mirrorurl.save(force_update=True)
+ # now check new values, only update if new != old
+ newvals = (mirrorurl.has_ipv4, mirrorurl.has_ipv6)
+ if newvals != oldvals:
+ logger.debug("values changed for %s", mirrorurl)
+ mirrorurl.save(update_fields=('has_ipv4', 'has_ipv6'))
except socket.error, e:
logger.warn("error resolving %s: %s", mirrorurl.hostname, e)
diff --git a/mirrors/migrations/0004_auto__add_field_mirrorprotocol_is_download.py b/mirrors/migrations/0004_auto__add_field_mirrorprotocol_is_download.py
index 5e44d211..0506e2cd 100644
--- a/mirrors/migrations/0004_auto__add_field_mirrorprotocol_is_download.py
+++ b/mirrors/migrations/0004_auto__add_field_mirrorprotocol_is_download.py
@@ -7,7 +7,7 @@ from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
- db.add_column('mirrors_mirrorprotocol', 'is_download', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=False)
+ db.add_column('mirrors_mirrorprotocol', 'is_download', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=True)
def backwards(self, orm):
db.delete_column('mirrors_mirrorprotocol', 'is_download')
diff --git a/mirrors/migrations/0006_auto__add_field_mirrorurl_has_ipv4__add_field_mirrorurl_has_ipv6.py b/mirrors/migrations/0006_auto__add_field_mirrorurl_has_ipv4__add_field_mirrorurl_has_ipv6.py
index a5e34589..5a40207d 100644
--- a/mirrors/migrations/0006_auto__add_field_mirrorurl_has_ipv4__add_field_mirrorurl_has_ipv6.py
+++ b/mirrors/migrations/0006_auto__add_field_mirrorurl_has_ipv4__add_field_mirrorurl_has_ipv6.py
@@ -7,8 +7,8 @@ from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
- db.add_column('mirrors_mirrorurl', 'has_ipv4', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=False)
- db.add_column('mirrors_mirrorurl', 'has_ipv6', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=False)
+ db.add_column('mirrors_mirrorurl', 'has_ipv4', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=True)
+ db.add_column('mirrors_mirrorurl', 'has_ipv6', self.gf('django.db.models.fields.BooleanField')(default=False), keep_default=True)
def backwards(self, orm):
db.delete_column('mirrors_mirrorurl', 'has_ipv4')
diff --git a/mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py b/mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py
index d30c78c7..66e60090 100644
--- a/mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py
+++ b/mirrors/migrations/0010_auto__add_field_mirrorprotocol_default.py
@@ -6,7 +6,7 @@ from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
- db.add_column('mirrors_mirrorprotocol', 'default', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=False)
+ db.add_column('mirrors_mirrorprotocol', 'default', self.gf('django.db.models.fields.BooleanField')(default=True), keep_default=True)
def backwards(self, orm):
db.delete_column('mirrors_mirrorprotocol', 'default')
diff --git a/mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py b/mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py
new file mode 100644
index 00000000..60c4ec26
--- /dev/null
+++ b/mirrors/migrations/0017_auto__chg_field_mirrorlog_error.py
@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+ def forwards(self, orm):
+ db.alter_column('mirrors_mirrorlog', 'error', self.gf('django.db.models.fields.TextField')(default=''))
+
+ def backwards(self, orm):
+ db.alter_column('mirrors_mirrorlog', 'error', self.gf('django.db.models.fields.CharField')(max_length=255))
+
+ models = {
+ 'mirrors.mirror': {
+ 'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'},
+ 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+ 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+ 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+ 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+ 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}),
+ 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'})
+ },
+ 'mirrors.mirrorlog': {
+ 'Meta': {'object_name': 'MirrorLog'},
+ 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+ 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
+ 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+ 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"})
+ },
+ 'mirrors.mirrorprotocol': {
+ 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'},
+ 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'})
+ },
+ 'mirrors.mirrorrsync': {
+ 'Meta': {'object_name': 'MirrorRsync'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}),
+ 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"})
+ },
+ 'mirrors.mirrorurl': {
+ 'Meta': {'object_name': 'MirrorUrl'},
+ 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}),
+ 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}),
+ 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}),
+ 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
+ }
+ }
+
+ complete_apps = ['mirrors']
diff --git a/mirrors/migrations/0018_auto__add_field_mirror_alternate_email.py b/mirrors/migrations/0018_auto__add_field_mirror_alternate_email.py
new file mode 100644
index 00000000..a08699e8
--- /dev/null
+++ b/mirrors/migrations/0018_auto__add_field_mirror_alternate_email.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+ def forwards(self, orm):
+ db.add_column('mirrors_mirror', 'alternate_email',
+ self.gf('django.db.models.fields.EmailField')(default='', max_length=255, blank=True),
+ keep_default=False)
+
+ def backwards(self, orm):
+ db.delete_column('mirrors_mirror', 'alternate_email')
+
+ models = {
+ 'mirrors.mirror': {
+ 'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'},
+ 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+ 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+ 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+ 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+ 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+ 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}),
+ 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'})
+ },
+ 'mirrors.mirrorlog': {
+ 'Meta': {'object_name': 'MirrorLog'},
+ 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+ 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
+ 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+ 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"})
+ },
+ 'mirrors.mirrorprotocol': {
+ 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'},
+ 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'})
+ },
+ 'mirrors.mirrorrsync': {
+ 'Meta': {'object_name': 'MirrorRsync'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}),
+ 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"})
+ },
+ 'mirrors.mirrorurl': {
+ 'Meta': {'object_name': 'MirrorUrl'},
+ 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}),
+ 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}),
+ 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}),
+ 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
+ }
+ }
+
+ complete_apps = ['mirrors']
diff --git a/mirrors/migrations/0019_move_country_data_to_url.py b/mirrors/migrations/0019_move_country_data_to_url.py
new file mode 100644
index 00000000..81b7bb3e
--- /dev/null
+++ b/mirrors/migrations/0019_move_country_data_to_url.py
@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import DataMigration
+from django.db import models
+
+class Migration(DataMigration):
+
+ def forwards(self, orm):
+ for url in orm.MirrorUrl.objects.select_related('mirror').all():
+ # set the country field on the URL if we have one,
+ # and it isn't already set to anything.
+ if url.country or not url.mirror.country:
+ continue
+ orm.MirrorUrl.objects.filter(pk=url.pk).update(
+ country=url.mirror.country)
+
+ def backwards(self, orm):
+ pass
+
+ models = {
+ 'mirrors.mirror': {
+ 'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'},
+ 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+ 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+ 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+ 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+ 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+ 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}),
+ 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'})
+ },
+ 'mirrors.mirrorlog': {
+ 'Meta': {'object_name': 'MirrorLog'},
+ 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+ 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
+ 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+ 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"})
+ },
+ 'mirrors.mirrorprotocol': {
+ 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'},
+ 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'})
+ },
+ 'mirrors.mirrorrsync': {
+ 'Meta': {'object_name': 'MirrorRsync'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}),
+ 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"})
+ },
+ 'mirrors.mirrorurl': {
+ 'Meta': {'object_name': 'MirrorUrl'},
+ 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}),
+ 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}),
+ 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}),
+ 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
+ }
+ }
+
+ complete_apps = ['mirrors']
+ symmetrical = True
diff --git a/mirrors/migrations/0020_auto__del_field_mirror_country.py b/mirrors/migrations/0020_auto__del_field_mirror_country.py
new file mode 100644
index 00000000..c2220a50
--- /dev/null
+++ b/mirrors/migrations/0020_auto__del_field_mirror_country.py
@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+ def forwards(self, orm):
+ db.delete_column('mirrors_mirror', 'country')
+
+
+ def backwards(self, orm):
+ db.add_column('mirrors_mirror', 'country',
+ self.gf('django_countries.fields.CountryField')(blank=True, default='', max_length=2, db_index=True),
+ keep_default=False)
+
+
+ models = {
+ 'mirrors.mirror': {
+ 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'},
+ 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+ 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+ 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+ 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+ 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}),
+ 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'})
+ },
+ 'mirrors.mirrorlog': {
+ 'Meta': {'object_name': 'MirrorLog'},
+ 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+ 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
+ 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+ 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"})
+ },
+ 'mirrors.mirrorprotocol': {
+ 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'},
+ 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'})
+ },
+ 'mirrors.mirrorrsync': {
+ 'Meta': {'object_name': 'MirrorRsync'},
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}),
+ 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"})
+ },
+ 'mirrors.mirrorurl': {
+ 'Meta': {'object_name': 'MirrorUrl'},
+ 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}),
+ 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}),
+ 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': "orm['mirrors.MirrorProtocol']"}),
+ 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
+ }
+ }
+
+ complete_apps = ['mirrors']
diff --git a/mirrors/migrations/0021_auto__chg_field_mirrorrsync_ip.py b/mirrors/migrations/0021_auto__chg_field_mirrorrsync_ip.py
new file mode 100644
index 00000000..bbf14bb0
--- /dev/null
+++ b/mirrors/migrations/0021_auto__chg_field_mirrorrsync_ip.py
@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+ def forwards(self, orm):
+ db.alter_column(u'mirrors_mirrorrsync', 'ip', self.gf('django.db.models.fields.CharField')(max_length=44))
+
+ def backwards(self, orm):
+ db.alter_column(u'mirrors_mirrorrsync', 'ip', self.gf('django.db.models.fields.CharField')(max_length=24))
+
+ models = {
+ u'mirrors.mirror': {
+ 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'},
+ 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+ 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+ 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+ 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+ 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}),
+ 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'})
+ },
+ u'mirrors.mirrorlog': {
+ 'Meta': {'object_name': 'MirrorLog'},
+ 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+ 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
+ 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+ 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"})
+ },
+ u'mirrors.mirrorprotocol': {
+ 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'},
+ 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'})
+ },
+ u'mirrors.mirrorrsync': {
+ 'Meta': {'object_name': 'MirrorRsync'},
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'ip': ('django.db.models.fields.CharField', [], {'max_length': '44'}),
+ 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"})
+ },
+ u'mirrors.mirrorurl': {
+ 'Meta': {'object_name': 'MirrorUrl'},
+ 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}),
+ 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}),
+ 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}),
+ 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
+ }
+ }
+
+ complete_apps = ['mirrors']
diff --git a/mirrors/migrations/0022_auto__add_checklocation.py b/mirrors/migrations/0022_auto__add_checklocation.py
new file mode 100644
index 00000000..896b2dab
--- /dev/null
+++ b/mirrors/migrations/0022_auto__add_checklocation.py
@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+ def forwards(self, orm):
+ db.create_table(u'mirrors_checklocation', (
+ (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+ ('hostname', self.gf('django.db.models.fields.CharField')(max_length=255)),
+ ('source_ip', self.gf('django.db.models.fields.GenericIPAddressField')(unique=True, max_length=39)),
+ ('country', self.gf('django_countries.fields.CountryField')(max_length=2)),
+ ('created', self.gf('django.db.models.fields.DateTimeField')()),
+ ))
+ db.send_create_signal(u'mirrors', ['CheckLocation'])
+
+
+ def backwards(self, orm):
+ db.delete_table(u'mirrors_checklocation')
+
+
+ models = {
+ u'mirrors.checklocation': {
+ 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'},
+ 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}),
+ 'created': ('django.db.models.fields.DateTimeField', [], {}),
+ 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'})
+ },
+ u'mirrors.mirror': {
+ 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'},
+ 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+ 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+ 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+ 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+ 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}),
+ 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'})
+ },
+ u'mirrors.mirrorlog': {
+ 'Meta': {'object_name': 'MirrorLog'},
+ 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+ 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
+ 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+ 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"})
+ },
+ u'mirrors.mirrorprotocol': {
+ 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'},
+ 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'})
+ },
+ u'mirrors.mirrorrsync': {
+ 'Meta': {'object_name': 'MirrorRsync'},
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'ip': ('django.db.models.fields.CharField', [], {'max_length': '44'}),
+ 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"})
+ },
+ u'mirrors.mirrorurl': {
+ 'Meta': {'object_name': 'MirrorUrl'},
+ 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}),
+ 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}),
+ 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}),
+ 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
+ }
+ }
+
+ complete_apps = ['mirrors']
diff --git a/mirrors/migrations/0023_auto__add_field_mirrorurl_created__add_field_mirrorrsync_created__add_.py b/mirrors/migrations/0023_auto__add_field_mirrorurl_created__add_field_mirrorrsync_created__add_.py
new file mode 100644
index 00000000..1a1f48a0
--- /dev/null
+++ b/mirrors/migrations/0023_auto__add_field_mirrorurl_created__add_field_mirrorrsync_created__add_.py
@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+from pytz import utc
+
+
+class Migration(SchemaMigration):
+
+ def forwards(self, orm):
+ default = datetime.datetime(2000, 1, 1, 0, 0).replace(tzinfo=utc)
+ db.add_column(u'mirrors_mirrorurl', 'created',
+ self.gf('django.db.models.fields.DateTimeField')(default=default),
+ keep_default=False)
+ db.add_column(u'mirrors_mirrorrsync', 'created',
+ self.gf('django.db.models.fields.DateTimeField')(default=default),
+ keep_default=False)
+ db.add_column(u'mirrors_mirrorprotocol', 'created',
+ self.gf('django.db.models.fields.DateTimeField')(default=default),
+ keep_default=False)
+ db.add_column(u'mirrors_mirror', 'created',
+ self.gf('django.db.models.fields.DateTimeField')(default=default),
+ keep_default=False)
+
+
+ def backwards(self, orm):
+ db.delete_column(u'mirrors_mirrorurl', 'created')
+ db.delete_column(u'mirrors_mirrorrsync', 'created')
+ db.delete_column(u'mirrors_mirrorprotocol', 'created')
+ db.delete_column(u'mirrors_mirror', 'created')
+
+
+ models = {
+ u'mirrors.checklocation': {
+ 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'},
+ 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}),
+ 'created': ('django.db.models.fields.DateTimeField', [], {}),
+ 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'})
+ },
+ u'mirrors.mirror': {
+ 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'},
+ 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+ 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+ 'created': ('django.db.models.fields.DateTimeField', [], {}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+ 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+ 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+ 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}),
+ 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'})
+ },
+ u'mirrors.mirrorlog': {
+ 'Meta': {'object_name': 'MirrorLog'},
+ 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+ 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
+ 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+ 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"})
+ },
+ u'mirrors.mirrorprotocol': {
+ 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'},
+ 'created': ('django.db.models.fields.DateTimeField', [], {}),
+ 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'})
+ },
+ u'mirrors.mirrorrsync': {
+ 'Meta': {'object_name': 'MirrorRsync'},
+ 'created': ('django.db.models.fields.DateTimeField', [], {}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'ip': ('django.db.models.fields.CharField', [], {'max_length': '44'}),
+ 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"})
+ },
+ u'mirrors.mirrorurl': {
+ 'Meta': {'object_name': 'MirrorUrl'},
+ 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}),
+ 'created': ('django.db.models.fields.DateTimeField', [], {}),
+ 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}),
+ 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}),
+ 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
+ }
+ }
+
+ complete_apps = ['mirrors']
diff --git a/mirrors/migrations/0024_auto__add_field_mirrorlog_location.py b/mirrors/migrations/0024_auto__add_field_mirrorlog_location.py
new file mode 100644
index 00000000..acf8df17
--- /dev/null
+++ b/mirrors/migrations/0024_auto__add_field_mirrorlog_location.py
@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+ def forwards(self, orm):
+ db.add_column(u'mirrors_mirrorlog', 'location',
+ self.gf('django.db.models.fields.related.ForeignKey')(related_name='logs', null=True, to=orm['mirrors.CheckLocation']),
+ keep_default=False)
+
+
+ def backwards(self, orm):
+ db.delete_column(u'mirrors_mirrorlog', 'location_id')
+
+
+ models = {
+ u'mirrors.checklocation': {
+ 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'},
+ 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}),
+ 'created': ('django.db.models.fields.DateTimeField', [], {}),
+ 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'})
+ },
+ u'mirrors.mirror': {
+ 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'},
+ 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+ 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+ 'created': ('django.db.models.fields.DateTimeField', [], {}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+ 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+ 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+ 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}),
+ 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'})
+ },
+ u'mirrors.mirrorlog': {
+ 'Meta': {'object_name': 'MirrorLog'},
+ 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+ 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
+ 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+ 'location': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'null': 'True', 'to': u"orm['mirrors.CheckLocation']"}),
+ 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"})
+ },
+ u'mirrors.mirrorprotocol': {
+ 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'},
+ 'created': ('django.db.models.fields.DateTimeField', [], {}),
+ 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'})
+ },
+ u'mirrors.mirrorrsync': {
+ 'Meta': {'object_name': 'MirrorRsync'},
+ 'created': ('django.db.models.fields.DateTimeField', [], {}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'ip': ('django.db.models.fields.CharField', [], {'max_length': '44'}),
+ 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"})
+ },
+ u'mirrors.mirrorurl': {
+ 'Meta': {'object_name': 'MirrorUrl'},
+ 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}),
+ 'created': ('django.db.models.fields.DateTimeField', [], {}),
+ 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}),
+ 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}),
+ 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
+ }
+ }
+
+ complete_apps = ['mirrors']
diff --git a/mirrors/migrations/0025_auto__chg_field_mirrorrsync_ip.py b/mirrors/migrations/0025_auto__chg_field_mirrorrsync_ip.py
new file mode 100644
index 00000000..b359b637
--- /dev/null
+++ b/mirrors/migrations/0025_auto__chg_field_mirrorrsync_ip.py
@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+ def forwards(self, orm):
+ if db.backend_name == 'postgres':
+ # For PostgreSQL, because it uses the 'inet' type and not a varchar
+ # column, we need to add an explict 'USING' cast to the SQL
+ # statement. We then execute the alter_column as well to ensure any
+ # of the other side-effects happen.
+ db.execute('ALTER TABLE "mirrors_mirrorrsync" ALTER COLUMN "ip" TYPE inet USING "ip"::inet')
+ db.alter_column(u'mirrors_mirrorrsync', 'ip', self.gf('mirrors.fields.IPNetworkField')(max_length=44))
+
+ def backwards(self, orm):
+ db.alter_column(u'mirrors_mirrorrsync', 'ip', self.gf('django.db.models.fields.CharField')(max_length=44))
+
+ models = {
+ u'mirrors.checklocation': {
+ 'Meta': {'ordering': "('hostname', 'source_ip')", 'object_name': 'CheckLocation'},
+ 'country': ('django_countries.fields.CountryField', [], {'max_length': '2'}),
+ 'created': ('django.db.models.fields.DateTimeField', [], {}),
+ 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'source_ip': ('django.db.models.fields.GenericIPAddressField', [], {'unique': 'True', 'max_length': '39'})
+ },
+ u'mirrors.mirror': {
+ 'Meta': {'ordering': "('name',)", 'object_name': 'Mirror'},
+ 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+ 'alternate_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
+ 'created': ('django.db.models.fields.DateTimeField', [], {}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
+ 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+ 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+ 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
+ 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}),
+ 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['mirrors.Mirror']", 'null': 'True', 'on_delete': 'models.SET_NULL'})
+ },
+ u'mirrors.mirrorlog': {
+ 'Meta': {'object_name': 'MirrorLog'},
+ 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
+ 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
+ 'error': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+ 'location': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'null': 'True', 'to': u"orm['mirrors.CheckLocation']"}),
+ 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': u"orm['mirrors.MirrorUrl']"})
+ },
+ u'mirrors.mirrorprotocol': {
+ 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'},
+ 'created': ('django.db.models.fields.DateTimeField', [], {}),
+ 'default': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'})
+ },
+ u'mirrors.mirrorrsync': {
+ 'Meta': {'object_name': 'MirrorRsync'},
+ 'created': ('django.db.models.fields.DateTimeField', [], {}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'ip': ('mirrors.fields.IPNetworkField', [], {'max_length': '44'}),
+ 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': u"orm['mirrors.Mirror']"})
+ },
+ u'mirrors.mirrorurl': {
+ 'Meta': {'object_name': 'MirrorUrl'},
+ 'country': ('django_countries.fields.CountryField', [], {'db_index': 'True', 'max_length': '2', 'blank': 'True'}),
+ 'created': ('django.db.models.fields.DateTimeField', [], {}),
+ 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+ 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+ u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+ 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': u"orm['mirrors.Mirror']"}),
+ 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'on_delete': 'models.PROTECT', 'to': u"orm['mirrors.MirrorProtocol']"}),
+ 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
+ }
+ }
+
+ complete_apps = ['mirrors']
diff --git a/mirrors/models.py b/mirrors/models.py
index 19437610..d8ac7952 100644
--- a/mirrors/models.py
+++ b/mirrors/models.py
@@ -1,55 +1,56 @@
import socket
from urlparse import urlparse
-from django.db import models
from django.core.exceptions import ValidationError
+from django.db import models
+from django.db.models.signals import pre_save
from django_countries import CountryField
-
-TIER_CHOICES = (
- (0, 'Tier 0'),
- (1, 'Tier 1'),
- (2, 'Tier 2'),
- (-1, 'Untiered'),
-)
+from .fields import IPNetworkField
+from main.utils import set_created_field
class Mirror(models.Model):
+ TIER_CHOICES = (
+ (0, 'Tier 0'),
+ (1, 'Tier 1'),
+ (2, 'Tier 2'),
+ (-1, 'Untiered'),
+ )
+
name = models.CharField(max_length=255, unique=True)
tier = models.SmallIntegerField(default=2, choices=TIER_CHOICES)
upstream = models.ForeignKey('self', null=True, on_delete=models.SET_NULL)
- country = CountryField(blank=True, db_index=True)
admin_email = models.EmailField(max_length=255, blank=True)
+ alternate_email = models.EmailField(max_length=255, blank=True)
public = models.BooleanField(default=True)
active = models.BooleanField(default=True)
isos = models.BooleanField("ISOs", default=True)
rsync_user = models.CharField(max_length=50, blank=True, default='')
rsync_password = models.CharField(max_length=50, blank=True, default='')
notes = models.TextField(blank=True)
+ created = models.DateTimeField(editable=False)
class Meta:
- ordering = ('country', 'name')
+ ordering = ('name',)
def __unicode__(self):
return self.name
- def supported_protocols(self):
- protocols = MirrorProtocol.objects.filter(
- urls__mirror=self).order_by('protocol').distinct()
- return sorted(protocols)
-
def downstream(self):
return Mirror.objects.filter(upstream=self).order_by('name')
def get_absolute_url(self):
return '/mirrors/%s/' % self.name
+
class MirrorProtocol(models.Model):
protocol = models.CharField(max_length=10, unique=True)
is_download = models.BooleanField(default=True,
help_text="Is protocol useful for end-users, e.g. FTP/HTTP")
default = models.BooleanField(default=True,
help_text="Included by default when building mirror list?")
+ created = models.DateTimeField(editable=False)
def __unicode__(self):
return self.protocol
@@ -57,6 +58,7 @@ class MirrorProtocol(models.Model):
class Meta:
ordering = ('protocol',)
+
class MirrorUrl(models.Model):
url = models.CharField("URL", max_length=255, unique=True)
protocol = models.ForeignKey(MirrorProtocol, related_name="urls",
@@ -67,6 +69,7 @@ class MirrorUrl(models.Model):
editable=False)
has_ipv6 = models.BooleanField("IPv6 capable", default=False,
editable=False)
+ created = models.DateTimeField(editable=False)
def address_families(self):
hostname = urlparse(self.url).hostname
@@ -78,10 +81,6 @@ class MirrorUrl(models.Model):
def hostname(self):
return urlparse(self.url).hostname
- @property
- def real_country(self):
- return self.country or self.mirror.country
-
def clean(self):
try:
# Auto-map the protocol field by looking at the URL
@@ -93,7 +92,7 @@ class MirrorUrl(models.Model):
families = self.address_families()
self.has_ipv4 = socket.AF_INET in families
self.has_ipv6 = socket.AF_INET6 in families
- except socket.error as e:
+ except socket.error:
# We don't fail in this case; we'll just set both to False
self.has_ipv4 = False
self.has_ipv6 = False
@@ -104,28 +103,69 @@ class MirrorUrl(models.Model):
class Meta:
verbose_name = 'mirror URL'
+
class MirrorRsync(models.Model):
- ip = models.CharField("IP", max_length=24)
+ # max length is 40 chars for full-form IPv6 addr + subnet
+ ip = IPNetworkField("IP")
mirror = models.ForeignKey(Mirror, related_name="rsync_ips")
+ created = models.DateTimeField(editable=False)
def __unicode__(self):
- return "%s" % (self.ip)
+ return self.ip
class Meta:
verbose_name = 'mirror rsync IP'
+
+class CheckLocation(models.Model):
+ hostname = models.CharField(max_length=255)
+ source_ip = models.GenericIPAddressField(verbose_name='source IP',
+ unpack_ipv4=True, unique=True)
+ country = CountryField()
+ created = models.DateTimeField(editable=False)
+
+ class Meta:
+ ordering = ('hostname', 'source_ip')
+
+ def __unicode__(self):
+ return self.hostname
+
+ @property
+ def family(self):
+ info = socket.getaddrinfo(self.source_ip, None, 0, 0, 0,
+ socket.AI_NUMERICHOST)
+ families = [x[0] for x in info]
+ return families[0]
+
+ @property
+ def ip_version(self):
+ '''Returns integer '4' or '6'.'''
+ if self.family == socket.AF_INET6:
+ return 6
+ if self.family == socket.AF_INET:
+ return 4
+ return None
+
+
class MirrorLog(models.Model):
url = models.ForeignKey(MirrorUrl, related_name="logs")
+ location = models.ForeignKey(CheckLocation, related_name="logs", null=True)
check_time = models.DateTimeField(db_index=True)
last_sync = models.DateTimeField(null=True)
duration = models.FloatField(null=True)
is_success = models.BooleanField(default=True)
- error = models.CharField(max_length=255, blank=True, default='')
+ error = models.TextField(blank=True, default='')
def __unicode__(self):
return "Check of %s at %s" % (self.url.url, self.check_time)
class Meta:
verbose_name = 'mirror check log'
+ get_latest_by = 'check_time'
+
+
+for model in (Mirror, MirrorProtocol, MirrorUrl, MirrorRsync, CheckLocation):
+ pre_save.connect(set_created_field, sender=model,
+ dispatch_uid="mirrors.models")
# vim: set ts=4 sw=4 et:
diff --git a/mirrors/static/mirror_status.js b/mirrors/static/mirror_status.js
new file mode 100644
index 00000000..241f5c61
--- /dev/null
+++ b/mirrors/static/mirror_status.js
@@ -0,0 +1,173 @@
+function draw_graphs(location_url, log_url, container_id) {
+ jQuery.when(jQuery.getJSON(location_url), jQuery.getJSON(log_url))
+ .then(function(loc_data, log_data) {
+ /* use the same color selection for a given URL in every graph */
+ var color = d3.scale.category10();
+ jQuery.each(loc_data[0].locations, function(i, val) {
+ mirror_status(container_id, val, log_data[0], color);
+ });
+ });
+}
+
+function mirror_status(container_id, check_loc, log_data, color) {
+
+ var draw_graph = function(chart_id, data) {
+ var jq_div = jQuery(chart_id);
+ var margin = {top: 20, right: 20, bottom: 30, left: 40},
+ width = jq_div.width() - margin.left - margin.right,
+ height = jq_div.height() - margin.top - margin.bottom;
+
+ var x = d3.time.scale.utc().range([0, width]),
+ y = d3.scale.linear().range([height, 0]),
+ x_axis = d3.svg.axis().scale(x).orient("bottom"),
+ y_axis = d3.svg.axis().scale(y).orient("left");
+
+ /* remove any existing graph first if we are redrawing after resize */
+ d3.select(chart_id).select("svg").remove();
+ var svg = d3.select(chart_id).append("svg")
+ .attr("width", width + margin.left + margin.right)
+ .attr("height", height + margin.top + margin.bottom)
+ .append("g")
+ .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
+
+ x.domain([
+ d3.min(data, function(c) { return d3.min(c.logs, function(v) { return v.check_time; }); }),
+ d3.max(data, function(c) { return d3.max(c.logs, function(v) { return v.check_time; }); })
+ ]).nice(d3.time.hour);
+ y.domain([
+ 0,
+ d3.max(data, function(c) { return d3.max(c.logs, function(v) { return v.duration; }); })
+ ]).nice();
+
+ /* build the axis lines... */
+ svg.append("g")
+ .attr("class", "x axis")
+ .attr("transform", "translate(0," + height + ")")
+ .call(x_axis)
+ .append("text")
+ .attr("class", "label")
+ .attr("x", width)
+ .attr("y", -6)
+ .style("text-anchor", "end")
+ .text("Check Time (UTC)");
+
+ svg.append("g")
+ .attr("class", "y axis")
+ .call(y_axis)
+ .append("text")
+ .attr("class", "label")
+ .attr("transform", "rotate(-90)")
+ .attr("y", 6)
+ .attr("dy", ".71em")
+ .style("text-anchor", "end")
+ .text("Duration (seconds)");
+
+ var line = d3.svg.line()
+ .interpolate("basis")
+ .x(function(d) { return x(d.check_time); })
+ .y(function(d) { return y(d.duration); });
+
+ /* ...then the points and lines between them. */
+ var urls = svg.selectAll(".url")
+ .data(data)
+ .enter()
+ .append("g")
+ .attr("class", "url");
+
+ urls.append("path")
+ .attr("class", "url-line")
+ .attr("d", function(d) { return line(d.logs); })
+ .style("stroke", function(d) { return color(d.url); });
+
+ urls.selectAll("circle")
+ .data(function(u) {
+ return jQuery.map(u.logs, function(l, i) {
+ return {url: u.url, check_time: l.check_time, duration: l.duration};
+ });
+ })
+ .enter()
+ .append("circle")
+ .attr("class", "url-dot")
+ .attr("r", 3.5)
+ .attr("cx", function(d) { return x(d.check_time); })
+ .attr("cy", function(d) { return y(d.duration); })
+ .style("fill", function(d) { return color(d.url); })
+ .append("title")
+ .text(function(d) { return d.url + "\n" + d.duration.toFixed(3) + " secs\n" + d.check_time.toUTCString(); });
+
+ /* add a legend for good measure */
+ var active = jQuery.map(data, function(item, i) { return item.url; });
+ var legend = svg.selectAll(".legend")
+ .data(active)
+ .enter().append("g")
+ .attr("class", "legend")
+ .attr("transform", function(d, i) { return "translate(0," + i * 20 + ")"; });
+
+ legend.append("rect")
+ .attr("x", width - 18)
+ .attr("width", 18)
+ .attr("height", 18)
+ .style("fill", color);
+
+ legend.append("text")
+ .attr("x", width - 24)
+ .attr("y", 9)
+ .attr("dy", ".35em")
+ .style("text-anchor", "end")
+ .text(function(d) { return d; });
+ };
+
+ var filter_data = function(json, location_id) {
+ return jQuery.map(json.urls, function(url, i) {
+ var logs = jQuery.map(url.logs, function(log, j) {
+ if (!log.is_success) {
+ return null;
+ }
+ /* screen by location ID if we were given one */
+ if (location_id && log.location_id !== location_id) {
+ return null;
+ }
+ return {
+ duration: log.duration,
+ check_time: new Date(log.check_time)
+ };
+ });
+ /* don't return URLs without any log info */
+ if (logs.length === 0) {
+ return null;
+ }
+ return {
+ url: url.url,
+ logs: logs
+ };
+ });
+ };
+
+ var cached_data = filter_data(log_data, check_loc.id);
+ /* we had a check location with no log data handed to us, skip graphing */
+ if (cached_data.length === 0) {
+ return;
+ }
+
+ /* create the containers, defer the actual graph drawing */
+ var chart_id = 'status-chart-' + check_loc.id;
+ jQuery(container_id).append('<h3><span class="fam-flag fam-flag-' + check_loc.country_code.toLowerCase() + '" title="' + check_loc.country + '"></span> ' + check_loc.country + ' (' + check_loc.source_ip + '), IPv' + check_loc.ip_version + '</h3>');
+ jQuery(container_id).append('<div id="' + chart_id + '" class="visualize-mirror visualize-chart"></div>');
+ jQuery(container_id).append('<br/>');
+ setTimeout(function() {
+ draw_graph('#' + chart_id, cached_data);
+ }, 0);
+
+ /* then hook up a resize handler to redraw if necessary */
+ var resize_timeout = null;
+ var real_resize = function() {
+ resize_timeout = null;
+ draw_graph('#' + chart_id, cached_data);
+ };
+ jQuery(window).resize(function() {
+ if (resize_timeout) {
+ clearTimeout(resize_timeout);
+ }
+ resize_timeout = setTimeout(real_resize, 200);
+ });
+}
diff --git a/mirrors/urls.py b/mirrors/urls.py
index f002e9d6..7cf76aa1 100644
--- a/mirrors/urls.py
+++ b/mirrors/urls.py
@@ -4,7 +4,11 @@ urlpatterns = patterns('mirrors.views',
(r'^$', 'mirrors', {}, 'mirror-list'),
(r'^status/$', 'status', {}, 'mirror-status'),
(r'^status/json/$', 'status_json', {}, 'mirror-status-json'),
+ (r'^status/tier/(?P<tier>\d+)/$', 'status', {}, 'mirror-status-tier'),
+ (r'^status/tier/(?P<tier>\d+)/json/$', 'status_json', {}, 'mirror-status-tier-json'),
+ (r'^locations/json/$', 'locations_json', {}, 'mirror-locations-json'),
(r'^(?P<name>[\.\-\w]+)/$', 'mirror_details'),
+ (r'^(?P<name>[\.\-\w]+)/json/$', 'mirror_details_json'),
)
# vim: set ts=4 sw=4 et:
diff --git a/mirrors/urls_mirrorlist.py b/mirrors/urls_mirrorlist.py
index e0f44c78..1444eca9 100644
--- a/mirrors/urls_mirrorlist.py
+++ b/mirrors/urls_mirrorlist.py
@@ -3,10 +3,7 @@ from django.conf.urls import patterns
urlpatterns = patterns('mirrors.views',
(r'^$', 'generate_mirrorlist', {}, 'mirrorlist'),
(r'^all/$', 'find_mirrors', {'countries': ['all']}),
- (r'^all/ftp/$', 'find_mirrors',
- {'countries': ['all'], 'protocols': ['ftp']}),
- (r'^all/http/$', 'find_mirrors',
- {'countries': ['all'], 'protocols': ['http']}),
+ (r'^all/(?P<protocol>[A-z]+)/$', 'find_mirrors_simple')
)
# vim: set ts=4 sw=4 et:
diff --git a/mirrors/utils.py b/mirrors/utils.py
index 32fa3587..ba45da5f 100644
--- a/mirrors/utils.py
+++ b/mirrors/utils.py
@@ -1,21 +1,107 @@
from datetime import timedelta
-from django.db.models import Avg, Count, Max, Min, StdDev
+from django.db import connection
+from django.db.models import Count, Max, Min
+from django.utils.dateparse import parse_datetime
+from django.utils.timezone import now
from django_countries.fields import Country
-from main.utils import cache_function, utc_now
-from .models import MirrorLog, MirrorProtocol, MirrorUrl
+from main.utils import cache_function, database_vendor
+from .models import MirrorLog, MirrorUrl
-default_cutoff = timedelta(hours=24)
+DEFAULT_CUTOFF = timedelta(hours=24)
-def annotate_url(url, delays):
+
+def dictfetchall(cursor):
+ "Returns all rows from a cursor as a dict."
+ desc = cursor.description
+ return [
+ dict(zip([col[0] for col in desc], row))
+ for row in cursor.fetchall()
+ ]
+
+
+def status_data(cutoff_time, mirror_id=None):
+ if mirror_id is not None:
+ params = [cutoff_time, mirror_id]
+ mirror_where = 'AND u.mirror_id = %s'
+ else:
+ params = [cutoff_time]
+ mirror_where = ''
+
+ vendor = database_vendor(MirrorUrl)
+ if vendor == 'sqlite':
+ sql = """
+SELECT l.url_id, u.mirror_id,
+ COUNT(l.id) AS check_count,
+ COUNT(l.duration) AS success_count,
+ MAX(l.last_sync) AS last_sync,
+ MAX(l.check_time) AS last_check,
+ AVG(l.duration) AS duration_avg,
+ 0.0 AS duration_stddev,
+ AVG(STRFTIME('%%s', check_time) - STRFTIME('%%s', last_sync)) AS delay
+FROM mirrors_mirrorlog l
+JOIN mirrors_mirrorurl u ON u.id = l.url_id
+WHERE l.check_time >= %s
+""" + mirror_where + """
+GROUP BY l.url_id, u.mirror_id
+"""
+ else:
+ sql = """
+SELECT l.url_id, u.mirror_id,
+ COUNT(l.id) AS check_count,
+ COUNT(l.duration) AS success_count,
+ MAX(l.last_sync) AS last_sync,
+ MAX(l.check_time) AS last_check,
+ AVG(l.duration) AS duration_avg,
+ STDDEV(l.duration) AS duration_stddev,
+ AVG(check_time - last_sync) AS delay
+FROM mirrors_mirrorlog l
+JOIN mirrors_mirrorurl u ON u.id = l.url_id
+WHERE l.check_time >= %s
+""" + mirror_where + """
+GROUP BY l.url_id, u.mirror_id
+"""
+
+ cursor = connection.cursor()
+ cursor.execute(sql, params)
+ url_data = dictfetchall(cursor)
+
+ # sqlite loves to return less than ideal types
+ if vendor == 'sqlite':
+ for item in url_data:
+ if item['delay'] is not None:
+ item['delay'] = timedelta(seconds=item['delay'])
+ if item['last_sync'] is not None:
+ item['last_sync'] = parse_datetime(item['last_sync'])
+ item['last_check'] = parse_datetime(item['last_check'])
+
+ return {item['url_id']: item for item in url_data}
+
+
+def annotate_url(url, url_data):
'''Given a MirrorURL object, add a few more attributes to it regarding
status, including completion_pct, delay, and score.'''
- url.completion_pct = float(url.success_count) / url.check_count
- if url.id in delays:
- url_delays = delays[url.id]
- url.delay = sum(url_delays, timedelta()) / len(url_delays)
+ known_attrs = (
+ ('success_count', 0),
+ ('check_count', 0),
+ ('completion_pct', None),
+ ('last_check', None),
+ ('last_sync', None),
+ ('delay', None),
+ ('score', None),
+ )
+ for k, v in known_attrs:
+ setattr(url, k, v)
+ for k, v in url_data.items():
+ if k not in ('url_id', 'mirror_id'):
+ setattr(url, k, v)
+
+ if url.check_count > 0:
+ url.completion_pct = float(url.success_count) / url.check_count
+
+ if url.delay is not None:
hours = url.delay.days * 24.0 + url.delay.seconds / 3600.0
if url.completion_pct > 0:
@@ -23,43 +109,33 @@ def annotate_url(url, delays):
else:
# arbitrary small value
divisor = 0.005
- url.score = (hours + url.duration_avg + url.duration_stddev) / divisor
- else:
- url.delay = None
- url.score = None
+ stddev = url.duration_stddev or 0.0
+ url.score = (hours + url.duration_avg + stddev) / divisor
-@cache_function(123)
-def get_mirror_statuses(cutoff=default_cutoff):
- cutoff_time = utc_now() - cutoff
- protocols = list(MirrorProtocol.objects.filter(is_download=True))
- # I swear, this actually has decent performance...
- urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter(
+def get_mirror_statuses(cutoff=DEFAULT_CUTOFF, mirror_id=None):
+ cutoff_time = now() - cutoff
+
+ valid_urls = MirrorUrl.objects.filter(
mirror__active=True, mirror__public=True,
- protocol__in=protocols,
- logs__check_time__gte=cutoff_time).annotate(
- check_count=Count('logs'),
- success_count=Count('logs__duration'),
- last_sync=Max('logs__last_sync'),
- last_check=Max('logs__check_time'),
- duration_avg=Avg('logs__duration'),
- duration_stddev=StdDev('logs__duration'))
-
- # The Django ORM makes it really hard to get actual average delay in the
- # above query, so run a seperate query for it and we will process the
- # results here.
- times = MirrorLog.objects.filter(is_success=True, last_sync__isnull=False,
- check_time__gte=cutoff_time)
- delays = {}
- for log in times:
- delay = log.check_time - log.last_sync
- delays.setdefault(log.url_id, []).append(delay)
+ logs__check_time__gte=cutoff_time).distinct()
+
+ if mirror_id:
+ valid_urls = valid_urls.filter(mirror_id=mirror_id)
+
+ url_data = status_data(cutoff_time, mirror_id)
+ urls = MirrorUrl.objects.select_related('mirror', 'protocol').filter(
+ id__in=valid_urls).order_by('mirror__id', 'url')
if urls:
- last_check = max([u.last_check for u in urls])
+ for url in urls:
+ annotate_url(url, url_data.get(url.id, {}))
+ last_check = max([u.last_check for u in urls if u.last_check])
num_checks = max([u.check_count for u in urls])
- check_info = MirrorLog.objects.filter(
- check_time__gte=cutoff_time).aggregate(
+ check_info = MirrorLog.objects.filter(check_time__gte=cutoff_time)
+ if mirror_id:
+ check_info = check_info.filter(url__mirror_id=mirror_id)
+ check_info = check_info.aggregate(
mn=Min('check_time'), mx=Max('check_time'))
if num_checks > 1:
check_frequency = (check_info['mx'] - check_info['mn']) \
@@ -71,9 +147,6 @@ def get_mirror_statuses(cutoff=default_cutoff):
num_checks = 0
check_frequency = None
- for url in urls:
- annotate_url(url, delays)
-
return {
'cutoff': cutoff,
'last_check': last_check,
@@ -83,29 +156,31 @@ def get_mirror_statuses(cutoff=default_cutoff):
}
-@cache_function(117)
-def get_mirror_errors(cutoff=default_cutoff):
- cutoff_time = utc_now() - cutoff
+def get_mirror_errors(cutoff=DEFAULT_CUTOFF, mirror_id=None):
+ cutoff_time = now() - cutoff
errors = MirrorLog.objects.filter(
is_success=False, check_time__gte=cutoff_time,
url__mirror__active=True, url__mirror__public=True).values(
'url__url', 'url__country', 'url__protocol__protocol',
- 'url__mirror__country', 'error').annotate(
+ 'url__mirror__tier', 'error').annotate(
error_count=Count('error'), last_occurred=Max('check_time')
).order_by('-last_occurred', '-error_count')
+
+ if mirror_id:
+ errors = errors.filter(url__mirror_id=mirror_id)
+
errors = list(errors)
for err in errors:
- ctry_code = err['url__country'] or err['url__mirror__country']
- err['country'] = Country(ctry_code)
+ err['country'] = Country(err['url__country'])
return errors
@cache_function(295)
-def get_mirror_url_for_download(cutoff=default_cutoff):
+def get_mirror_url_for_download(cutoff=DEFAULT_CUTOFF):
'''Find a good mirror URL to use for package downloads. If we have mirror
status data available, it is used to determine a good choice by looking at
the last batch of status rows.'''
- cutoff_time = utc_now() - cutoff
+ cutoff_time = now() - cutoff
status_data = MirrorLog.objects.filter(
check_time__gte=cutoff_time).aggregate(
Max('check_time'), Max('last_sync'))
@@ -123,7 +198,7 @@ def get_mirror_url_for_download(cutoff=default_cutoff):
mirror_urls = MirrorUrl.objects.filter(
mirror__public=True, mirror__active=True, protocol__default=True)
# look first for a country-agnostic URL, then fall back to any HTTP URL
- filtered_urls = mirror_urls.filter(mirror__country='')[:1]
+ filtered_urls = mirror_urls.filter(country='')[:1]
if not filtered_urls:
filtered_urls = mirror_urls[:1]
if not filtered_urls:
diff --git a/mirrors/views.py b/mirrors/views.py
index c52656f7..73d40297 100644
--- a/mirrors/views.py
+++ b/mirrors/views.py
@@ -1,4 +1,6 @@
from datetime import timedelta
+from itertools import groupby
+import json
from operator import attrgetter, itemgetter
from django import forms
@@ -6,16 +8,14 @@ from django.forms.widgets import CheckboxSelectMultiple
from django.core.serializers.json import DjangoJSONEncoder
from django.db.models import Q
from django.http import Http404, HttpResponse
-from django.shortcuts import get_object_or_404
+from django.shortcuts import get_object_or_404, render
+from django.utils.timezone import now
from django.views.decorators.csrf import csrf_exempt
-from django.views.generic.simple import direct_to_template
-from django.utils import simplejson
from django_countries.countries import COUNTRIES
-from .models import Mirror, MirrorUrl, MirrorProtocol
-from .utils import get_mirror_statuses, get_mirror_errors
-
-COUNTRY_LOOKUP = dict(COUNTRIES)
+from .models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog,
+ CheckLocation)
+from .utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF
class MirrorlistForm(forms.Form):
@@ -27,6 +27,8 @@ class MirrorlistForm(forms.Form):
widget=CheckboxSelectMultiple)
use_mirror_status = forms.BooleanField(required=False)
+ countries = dict(COUNTRIES)
+
def __init__(self, *args, **kwargs):
super(MirrorlistForm, self).__init__(*args, **kwargs)
fields = self.fields
@@ -41,13 +43,10 @@ class MirrorlistForm(forms.Form):
def get_countries(self):
country_codes = set()
- country_codes.update(Mirror.objects.filter(active=True).exclude(
- country='').values_list(
- 'country', flat=True).order_by().distinct())
country_codes.update(MirrorUrl.objects.filter(
mirror__active=True).exclude(country='').values_list(
'country', flat=True).order_by().distinct())
- countries = [(code, COUNTRY_LOOKUP[code]) for code in country_codes]
+ countries = [(code, self.countries[code]) for code in country_codes]
return sorted(countries, key=itemgetter(1))
def as_div(self):
@@ -75,22 +74,50 @@ def generate_mirrorlist(request):
else:
form = MirrorlistForm()
- return direct_to_template(request, 'mirrors/mirrorlist_generate.html',
+ return render(request, 'mirrors/mirrorlist_generate.html',
{'mirrorlist_form': form})
+def default_protocol_filter(original_urls):
+ key_func = attrgetter('country')
+ sorted_urls = sorted(original_urls, key=key_func)
+ urls = []
+ for _, group in groupby(sorted_urls, key=key_func):
+ cntry_urls = list(group)
+ if any(url.protocol.default for url in cntry_urls):
+ cntry_urls = [url for url in cntry_urls if url.protocol.default]
+ urls.extend(cntry_urls)
+ return urls
+
+
+def status_filter(original_urls):
+ status_info = get_mirror_statuses()
+ scores = {u.id: u.score for u in status_info['urls']}
+ urls = []
+ for u in original_urls:
+ u.score = scores.get(u.id, None)
+ # also include mirrors that don't have an up to date score
+ # (as opposed to those that have been set with no score)
+ if (u.id not in scores) or (u.score and u.score < 100.0):
+ urls.append(u)
+ # if a url doesn't have a score, treat it as the highest possible
+ return sorted(urls, key=lambda x: x.score or 100.0)
+
+
def find_mirrors(request, countries=None, protocols=None, use_status=False,
- ipv4_supported=True, ipv6_supported=True):
+ ipv4_supported=True, ipv6_supported=True, smart_protocol=False):
if not protocols:
- protocols = MirrorProtocol.objects.filter(
- is_download=True).values_list('protocol', flat=True)
+ protocols = MirrorProtocol.objects.filter(is_download=True)
+ elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol:
+ # we already have a queryset, no need to query again
+ pass
+ else:
+ protocols = MirrorProtocol.objects.filter(protocol__in=protocols)
qset = MirrorUrl.objects.select_related().filter(
- protocol__protocol__in=protocols,
- mirror__public=True, mirror__active=True,
- )
+ protocol__in=protocols,
+ mirror__public=True, mirror__active=True)
if countries and 'all' not in countries:
- qset = qset.filter(Q(country__in=countries) |
- Q(mirror__country__in=countries))
+ qset = qset.filter(country__in=countries)
ip_version = Q()
if ipv4_supported:
@@ -99,36 +126,47 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False,
ip_version |= Q(has_ipv6=True)
qset = qset.filter(ip_version)
+ if smart_protocol:
+ urls = default_protocol_filter(qset)
+ else:
+ urls = qset
+
if not use_status:
- urls = qset.order_by('mirror__name', 'url')
- urls = sorted(urls, key=attrgetter('real_country'))
+ sort_key = attrgetter('country.name', 'mirror.name', 'url')
+ urls = sorted(urls, key=sort_key)
template = 'mirrors/mirrorlist.txt'
else:
- status_info = get_mirror_statuses()
- scores = dict([(u.id, u.score) for u in status_info['urls']])
- urls = []
- for u in qset:
- u.score = scores.get(u.id, None)
- # also include mirrors that don't have an up to date score
- # (as opposed to those that have been set with no score)
- if (u.id not in scores) or \
- (u.score and u.score < 100.0):
- urls.append(u)
- # if a url doesn't have a score, treat it as the highest possible
- urls = sorted(urls, key=lambda x: x.score or 100.0)
+ urls = status_filter(urls)
template = 'mirrors/mirrorlist_status.txt'
- return direct_to_template(request, template, {
- 'mirror_urls': urls,
- },
- mimetype='text/plain')
+ context = {
+ 'mirror_urls': urls,
+ }
+ return render(request, template, context, content_type='text/plain')
+
+
+def find_mirrors_simple(request, protocol):
+ if protocol == 'smart':
+ # generate a 'smart' mirrorlist, one that only includes FTP mirrors if
+ # no HTTP mirror is available in that country.
+ return find_mirrors(request, smart_protocol=True)
+ proto = get_object_or_404(MirrorProtocol, protocol=protocol)
+ return find_mirrors(request, protocols=[proto])
def mirrors(request):
- mirror_list = Mirror.objects.select_related().order_by('tier', 'country')
+ mirror_list = Mirror.objects.select_related().order_by('tier', 'name')
+ protos = MirrorUrl.objects.values_list(
+ 'mirror_id', 'protocol__protocol').order_by(
+ 'mirror__id', 'protocol__protocol').distinct()
if not request.user.is_authenticated():
mirror_list = mirror_list.filter(public=True, active=True)
- return direct_to_template(request, 'mirrors/mirrors.html',
+ protos = protos.filter(mirror__public=True, mirror__active=True)
+ protos = {k: list(v) for k, v in groupby(protos, key=itemgetter(0))}
+ for mirror in mirror_list:
+ items = protos.get(mirror.id, [])
+ mirror.protocols = [item[1] for item in items]
+ return render(request, 'mirrors/mirrors.html',
{'mirror_list': mirror_list})
@@ -138,20 +176,38 @@ def mirror_details(request, name):
(not mirror.public or not mirror.active):
raise Http404
- status_info = get_mirror_statuses()
- checked_urls = [url for url in status_info['urls'] \
- if url.mirror_id == mirror.id]
- all_urls = mirror.urls.select_related('protocol')
- # get each item from checked_urls and supplement with anything in all_urls
- # if it wasn't there
- all_urls = set(checked_urls).union(all_urls)
- all_urls = sorted(all_urls, key=attrgetter('url'))
-
- return direct_to_template(request, 'mirrors/mirror_details.html',
+ status_info = get_mirror_statuses(mirror_id=mirror.id)
+ checked_urls = {url for url in status_info['urls'] \
+ if url.mirror_id == mirror.id}
+ all_urls = set(mirror.urls.select_related('protocol'))
+ # Add dummy data for URLs that we haven't checked recently
+ other_urls = all_urls.difference(checked_urls)
+ for url in other_urls:
+ for attr in ('last_sync', 'completion_pct', 'delay', 'duration_avg',
+ 'duration_stddev', 'score'):
+ setattr(url, attr, None)
+ all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url'))
+
+ return render(request, 'mirrors/mirror_details.html',
{'mirror': mirror, 'urls': all_urls})
-def status(request):
+def mirror_details_json(request, name):
+ mirror = get_object_or_404(Mirror, name=name)
+ status_info = get_mirror_statuses(mirror_id=mirror.id)
+ data = status_info.copy()
+ data['version'] = 3
+ to_json = json.dumps(data, ensure_ascii=False,
+ cls=ExtendedMirrorStatusJSONEncoder)
+ response = HttpResponse(to_json, content_type='application/json')
+ return response
+
+
+def status(request, tier=None):
+ if tier is not None:
+ tier = int(tier)
+ if tier not in [t[0] for t in Mirror.TIER_CHOICES]:
+ raise Http404
bad_timedelta = timedelta(days=3)
status_info = get_mirror_statuses()
@@ -159,26 +215,35 @@ def status(request):
good_urls = []
bad_urls = []
for url in urls:
+ # screen by tier if we were asked to
+ if tier is not None and url.mirror.tier != tier:
+ continue
# split them into good and bad lists based on delay
if not url.delay or url.delay > bad_timedelta:
bad_urls.append(url)
else:
good_urls.append(url)
+ error_logs = get_mirror_errors()
+ if tier is not None:
+ error_logs = [log for log in error_logs
+ if log['url__mirror__tier'] == tier]
+
context = status_info.copy()
context.update({
'good_urls': sorted(good_urls, key=attrgetter('score')),
'bad_urls': sorted(bad_urls, key=lambda u: u.delay or timedelta.max),
- 'error_logs': get_mirror_errors(),
+ 'error_logs': error_logs,
+ 'tier': tier,
})
- return direct_to_template(request, 'mirrors/status.html', context)
+ return render(request, 'mirrors/status.html', context)
class MirrorStatusJSONEncoder(DjangoJSONEncoder):
'''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl
serialization. The base class takes care of datetime.datetime types.'''
- url_attributes = ['url', 'protocol', 'last_sync', 'completion_pct',
- 'delay', 'duration_avg', 'duration_stddev', 'score']
+ url_attributes = ('url', 'protocol', 'last_sync', 'completion_pct',
+ 'delay', 'duration_avg', 'duration_stddev', 'score')
def default(self, obj):
if isinstance(obj, timedelta):
@@ -188,10 +253,8 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder):
# mainly for queryset serialization
return list(obj)
if isinstance(obj, MirrorUrl):
- data = dict((attr, getattr(obj, attr))
- for attr in self.url_attributes)
- # get any override on the country attribute first
- country = obj.real_country
+ data = {attr: getattr(obj, attr) for attr in self.url_attributes}
+ country = obj.country
data['country'] = unicode(country.name)
data['country_code'] = country.code
return data
@@ -200,13 +263,63 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder):
return super(MirrorStatusJSONEncoder, self).default(obj)
-def status_json(request):
+class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder):
+ '''Adds URL check history information.'''
+ log_attributes = ('check_time', 'last_sync', 'duration', 'is_success',
+ 'location_id')
+
+ def default(self, obj):
+ if isinstance(obj, MirrorUrl):
+ data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj)
+ cutoff = now() - DEFAULT_CUTOFF
+ data['logs'] = obj.logs.filter(
+ check_time__gte=cutoff).order_by('check_time')
+ return data
+ if isinstance(obj, MirrorLog):
+ return {attr: getattr(obj, attr) for attr in self.log_attributes}
+ return super(ExtendedMirrorStatusJSONEncoder, self).default(obj)
+
+
+def status_json(request, tier=None):
+ if tier is not None:
+ tier = int(tier)
+ if tier not in [t[0] for t in Mirror.TIER_CHOICES]:
+ raise Http404
status_info = get_mirror_statuses()
data = status_info.copy()
+ if tier is not None:
+ data['urls'] = [url for url in data['urls'] if url.mirror.tier == tier]
data['version'] = 3
- to_json = simplejson.dumps(data, ensure_ascii=False,
- cls=MirrorStatusJSONEncoder)
- response = HttpResponse(to_json, mimetype='application/json')
+ to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder)
+ response = HttpResponse(to_json, content_type='application/json')
+ return response
+
+
+class LocationJSONEncoder(DjangoJSONEncoder):
+ '''Base JSONEncoder extended to handle CheckLocation objects.'''
+
+ def default(self, obj):
+ if hasattr(obj, '__iter__'):
+ # mainly for queryset serialization
+ return list(obj)
+ if isinstance(obj, CheckLocation):
+ return {
+ 'id': obj.pk,
+ 'hostname': obj.hostname,
+ 'source_ip': obj.source_ip,
+ 'country': unicode(obj.country.name),
+ 'country_code': obj.country.code,
+ 'ip_version': obj.ip_version,
+ }
+ return super(LocationJSONEncoder, self).default(obj)
+
+
+def locations_json(request):
+ data = {}
+ data['version'] = 1
+ data['locations'] = CheckLocation.objects.all().order_by('pk')
+ to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder)
+ response = HttpResponse(to_json, content_type='application/json')
return response
# vim: set ts=4 sw=4 et: