From 7c26f6b7a4d29faede58d2feb13ef961e4725637 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 19 Oct 2014 14:08:15 -0500 Subject: Use raw DB query to fetch last modified date Signed-off-by: Dan McGee --- mirrors/views.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index 26b5b802..55c40c4d 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -6,6 +6,7 @@ from operator import attrgetter, itemgetter from django import forms from django.forms.widgets import CheckboxSelectMultiple from django.core.serializers.json import DjangoJSONEncoder +from django.db import connection from django.db.models import Q from django.http import Http404, HttpResponse from django.shortcuts import get_object_or_404, redirect, render @@ -222,7 +223,9 @@ def url_details(request, name, url_id): def status_last_modified(request, *args, **kwargs): - return MirrorLog.objects.values_list('check_time', flat=True).latest() + cursor = connection.cursor() + cursor.execute("SELECT MAX(check_time) FROM mirrors_mirrorlog") + return cursor.fetchone()[0] @condition(last_modified_func=status_last_modified) -- cgit v1.2.3-2-g168b From ee6bf2782068b917232c71189aea0011b47e876d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 21 Oct 2014 09:10:28 -0500 Subject: Small performance tweaks to mirror status JSON encoding Do a few things to speed up the encoding of the JSON, including better usage of list comprehensions, less dynamic setattr() usage, and removal of the queryset specialization since we can easily do it outside of the encoder. Signed-off-by: Dan McGee --- mirrors/views.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index 55c40c4d..0bf0a267 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -275,9 +275,6 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder): if isinstance(obj, timedelta): # always returned as integer seconds return obj.days * 24 * 3600 + obj.seconds - if hasattr(obj, '__iter__'): - # mainly for queryset serialization - return list(obj) if isinstance(obj, MirrorUrl): data = {attr: getattr(obj, attr) for attr in self.url_attributes} country = obj.country @@ -298,8 +295,8 @@ class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): if isinstance(obj, MirrorUrl): data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj) cutoff = now() - DEFAULT_CUTOFF - data['logs'] = obj.logs.filter( - check_time__gte=cutoff).order_by('check_time') + data['logs'] = list(obj.logs.filter( + check_time__gte=cutoff).order_by('check_time')) return data if isinstance(obj, MirrorLog): return {attr: getattr(obj, attr) for attr in self.log_attributes} -- cgit v1.2.3-2-g168b From 087b4b00031fed52eeddf05ae36825cb498680f0 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 21 Oct 2014 09:11:48 -0500 Subject: Remove queryset specialization in JSON encoder Signed-off-by: Dan McGee --- mirrors/views.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index 0bf0a267..1a9741ed 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -323,9 +323,6 @@ class LocationJSONEncoder(DjangoJSONEncoder): '''Base JSONEncoder extended to handle CheckLocation objects.''' def default(self, obj): - if hasattr(obj, '__iter__'): - # mainly for queryset serialization - return list(obj) if isinstance(obj, CheckLocation): return { 'id': obj.pk, @@ -341,7 +338,7 @@ class LocationJSONEncoder(DjangoJSONEncoder): def locations_json(request): data = {} data['version'] = 1 - data['locations'] = CheckLocation.objects.all().order_by('pk') + data['locations'] = list(CheckLocation.objects.all().order_by('pk')) to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder) response = HttpResponse(to_json, content_type='application/json') return response -- cgit v1.2.3-2-g168b From 86fd0b722afb53670ef9a155a3c55d688f275c6d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 21 Oct 2014 09:39:06 -0500 Subject: Reduce complexity of status data URL query Get rid of all the junk trying to only return URLs that have been checked in the last 24 hours; it just isn't worth it. Instead, do that screening only in the views that need it, namely the HTML status page. Signed-off-by: Dan McGee --- mirrors/views.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index 1a9741ed..90787763 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -245,7 +245,10 @@ def status(request, tier=None): if tier is not None and url.mirror.tier != tier: continue # split them into good and bad lists based on delay - if not url.delay or url.delay > bad_timedelta: + if url.completion_pct is None: + # skip URLs that have never been checked + continue + elif not url.delay or url.delay > bad_timedelta: bad_urls.append(url) else: good_urls.append(url) -- cgit v1.2.3-2-g168b From c86ef5c326212f09a22f5ae3502a0bc79033a23a Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 21 Oct 2014 17:08:56 -0500 Subject: Use cache_page on mirror status JSON Signed-off-by: Dan McGee --- mirrors/views.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index 90787763..c2736da8 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -11,6 +11,7 @@ from django.db.models import Q from django.http import Http404, HttpResponse from django.shortcuts import get_object_or_404, redirect, render from django.utils.timezone import now +from django.views.decorators.cache import cache_page from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import condition from django_countries.data import COUNTRIES @@ -306,6 +307,7 @@ class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder): return super(ExtendedMirrorStatusJSONEncoder, self).default(obj) +@cache_page(67) @condition(last_modified_func=status_last_modified) def status_json(request, tier=None): if tier is not None: -- cgit v1.2.3-2-g168b From d009cd0b5c4ba018681942daa59452fe2b1a42c2 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 21 Oct 2014 20:33:12 -0500 Subject: Fix error when viewing mirror with no active URLs Signed-off-by: Dan McGee --- mirrors/views.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'mirrors/views.py') diff --git a/mirrors/views.py b/mirrors/views.py index c2736da8..cffafbf5 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -193,6 +193,8 @@ def mirror_details(request, name): def mirror_details_json(request, name): authorized = request.user.is_authenticated() mirror = get_object_or_404(Mirror, name=name) + if not authorized and (not mirror.public or not mirror.active): + raise Http404 status_info = get_mirror_statuses(mirror_id=mirror.id, show_all=authorized) data = status_info.copy() -- cgit v1.2.3-2-g168b