From fc7fc1d10a13e9b21d222435ba12affd1d08822f Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 16 Feb 2011 21:51:23 -0600 Subject: Slight tweaks to mirror commands We know we are doing updates when setting IPv4/IPv6 information, so set force_update to True to save the useless select query on each save(). For mirror checks, use a less cumbersome deque for the results since it is also thread-safe, and have all the log entries committed in one go. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 17 ++++++++--------- mirrors/management/commands/mirrorresolv.py | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) (limited to 'mirrors/management/commands') diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 44d2b364..51be71ea 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -10,7 +10,9 @@ Usage: ./manage.py mirrorcheck """ from django.core.management.base import NoArgsCommand +from django.db import transaction +from collections import deque from datetime import datetime, timedelta import logging import re @@ -130,7 +132,7 @@ def mirror_url_worker(work, output): item = work.get(block=False) try: log = check_mirror_url(item) - output.put(log) + output.append(log) finally: work.task_done() except Empty: @@ -139,7 +141,7 @@ def mirror_url_worker(work, output): class MirrorCheckPool(object): def __init__(self, work, num_threads=10): self.tasks = Queue() - self.logs = Queue() + self.logs = deque() for i in list(work): self.tasks.put(i) self.threads = [] @@ -149,6 +151,7 @@ class MirrorCheckPool(object): thread.daemon = True self.threads.append(thread) + @transaction.commit_on_success def run(self): logger.debug("starting threads") for t in self.threads: @@ -156,13 +159,9 @@ class MirrorCheckPool(object): logger.debug("joining on all threads") self.tasks.join() logger.debug("processing log entries") - try: - while True: - log = self.logs.get(block=False) - log.save() - self.logs.task_done() - except Empty: - logger.debug("all log items saved to database") + for log in self.logs: + log.save() + logger.debug("log entries saved") def check_current_mirrors(): urls = MirrorUrl.objects.filter( diff --git a/mirrors/management/commands/mirrorresolv.py b/mirrors/management/commands/mirrorresolv.py index 8cbd51a3..77e7a223 100644 --- a/mirrors/management/commands/mirrorresolv.py +++ b/mirrors/management/commands/mirrorresolv.py @@ -50,7 +50,7 @@ def resolve_mirrors(): mirrorurl.has_ipv6 = socket.AF_INET6 in families logger.debug("%s: v4: %s v6: %s", hostname, mirrorurl.has_ipv4, mirrorurl.has_ipv6) - mirrorurl.save() + mirrorurl.save(force_update=True) except socket.error, e: logger.warn("error resolving %s: %s", hostname, e) -- cgit v1.2.3-2-g168b From 6d5909ca5dc8934cacd969e964573989dc6b0417 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 27 Feb 2011 12:44:30 -0600 Subject: Auto-resolve mirror URLs on save This prevents people from having to mess with these checkboxes at all in the admin, and we incur no delay on their initial values being correct waiting for the cron job to run. Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorresolv.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'mirrors/management/commands') diff --git a/mirrors/management/commands/mirrorresolv.py b/mirrors/management/commands/mirrorresolv.py index 77e7a223..8a628bd4 100644 --- a/mirrors/management/commands/mirrorresolv.py +++ b/mirrors/management/commands/mirrorresolv.py @@ -12,7 +12,6 @@ from django.core.management.base import NoArgsCommand import sys import logging -from urlparse import urlparse import socket from mirrors.models import MirrorUrl @@ -42,13 +41,11 @@ def resolve_mirrors(): logger.debug("requesting list of mirror URLs") for mirrorurl in MirrorUrl.objects.filter(mirror__active=True): try: - hostname = urlparse(mirrorurl.url).hostname - logger.debug("resolving %3i (%s)", mirrorurl.id, hostname) - info = socket.getaddrinfo(hostname, None, 0, socket.SOCK_STREAM) - families = [x[0] for x in info] + logger.debug("resolving %3i (%s)", mirrorurl.id, mirrorurl.hostname) + families = mirrorurl.address_families() mirrorurl.has_ipv4 = socket.AF_INET in families mirrorurl.has_ipv6 = socket.AF_INET6 in families - logger.debug("%s: v4: %s v6: %s", hostname, + logger.debug("%s: v4: %s v6: %s", mirrorurl.hostname, mirrorurl.has_ipv4, mirrorurl.has_ipv6) mirrorurl.save(force_update=True) except socket.error, e: -- cgit v1.2.3-2-g168b