From 89c6ffc95cb1d5fe4bd2534562ca732d727a8686 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 20 Sep 2012 09:34:46 -0500 Subject: chmod -x reporead_inotify.py Signed-off-by: Dan McGee --- devel/management/commands/reporead_inotify.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 devel/management/commands/reporead_inotify.py (limited to 'devel/management/commands/reporead_inotify.py') diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py old mode 100755 new mode 100644 -- cgit v1.2.3-2-g168b From 3530303c9a7d017bdfec40d9dc7c38bd3fb2c09b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 25 Sep 2012 18:21:10 -0500 Subject: Only watch non-staging repos in inotify reporead This is temporary until we do more work to ensure staging packages don't show up and confuse regular users of the web interface. Signed-off-by: Dan McGee --- devel/management/commands/reporead_inotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'devel/management/commands/reporead_inotify.py') diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py index c74762eb..043e13fe 100644 --- a/devel/management/commands/reporead_inotify.py +++ b/devel/management/commands/reporead_inotify.py @@ -68,7 +68,7 @@ class Command(BaseCommand): and passes these on to the various pyinotify pieces as necessary and finally builds and returns a notifier object.''' arches = Arch.objects.filter(agnostic=False) - repos = Repo.objects.all() + repos = Repo.objects.filter(staging=False) arch_path_map = dict((arch, None) for arch in arches) all_paths = set() total_paths = 0 -- cgit v1.2.3-2-g168b From ed1adeb1254c4d5754260bbe1ae2fbbc2a88debb Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 30 Sep 2012 02:02:04 -0500 Subject: Begin importing staging repos This reverts 3530303c9a7d now that we have reasonably hidden most staging package confusion on the site for normal end users. Signed-off-by: Dan McGee --- devel/management/commands/reporead_inotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'devel/management/commands/reporead_inotify.py') diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py index 043e13fe..c74762eb 100644 --- a/devel/management/commands/reporead_inotify.py +++ b/devel/management/commands/reporead_inotify.py @@ -68,7 +68,7 @@ class Command(BaseCommand): and passes these on to the various pyinotify pieces as necessary and finally builds and returns a notifier object.''' arches = Arch.objects.filter(agnostic=False) - repos = Repo.objects.filter(staging=False) + repos = Repo.objects.all() arch_path_map = dict((arch, None) for arch in arches) all_paths = set() total_paths = 0 -- cgit v1.2.3-2-g168b From 6dd4d54bb0adbbb0f8c2b1beaa92b7a58971cf88 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 16 Nov 2012 16:20:11 -0600 Subject: Use Python 2.7 dictionary comprehension syntax Rather than the old idiom of dict((k, v) for <> in <>). Signed-off-by: Dan McGee --- devel/management/commands/reporead_inotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'devel/management/commands/reporead_inotify.py') diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py index c74762eb..16b3869c 100644 --- a/devel/management/commands/reporead_inotify.py +++ b/devel/management/commands/reporead_inotify.py @@ -69,7 +69,7 @@ class Command(BaseCommand): finally builds and returns a notifier object.''' arches = Arch.objects.filter(agnostic=False) repos = Repo.objects.all() - arch_path_map = dict((arch, None) for arch in arches) + arch_path_map = {arch: None for arch in arches} all_paths = set() total_paths = 0 for arch in arches: -- cgit v1.2.3-2-g168b From 9e9157d0a8cbf9ea076231e438fb30f58bff8e29 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 16 Nov 2012 16:37:31 -0600 Subject: Use python set comprehension syntax supported in 2.7 Signed-off-by: Dan McGee --- devel/management/commands/reporead_inotify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'devel/management/commands/reporead_inotify.py') diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py index 16b3869c..04f65764 100644 --- a/devel/management/commands/reporead_inotify.py +++ b/devel/management/commands/reporead_inotify.py @@ -77,7 +77,7 @@ class Command(BaseCommand): for repo in repos) # take a python format string and generate all unique combinations # of directories from it; using set() ensures we filter it down - paths = set(self.path_template % values for values in combos) + paths = {self.path_template % values for values in combos} total_paths += len(paths) all_paths |= paths arch_path_map[arch] = paths -- cgit v1.2.3-2-g168b From af32c23768c7537f19e0613525579208b4f44eb4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 15 Jan 2013 20:49:56 -0600 Subject: Handle connection and transaction more properly in reporead A few minor things are fixed here. One is PostgreSQL, and more specifically pgbouncer, don't like it when the connection is closed after psycopg2 has started an implicit transaction even for read-only queries. Ensure we call commit as our last database action in all cases. The other is related- Django in management commands doesn't ever call close on any database connection you may have been using, so PostgreSQL gets mad about this fact and logs a message saying such. Close the connection explicitly when we are done with it to play nice. Signed-off-by: Dan McGee --- devel/management/commands/reporead_inotify.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'devel/management/commands/reporead_inotify.py') diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py index 04f65764..8c1e47bf 100644 --- a/devel/management/commands/reporead_inotify.py +++ b/devel/management/commands/reporead_inotify.py @@ -23,7 +23,7 @@ import threading import time from django.core.management.base import BaseCommand, CommandError -from django.db import connection +from django.db import connection, transaction from main.models import Arch, Repo from .reporead import read_repo @@ -53,6 +53,11 @@ class Command(BaseCommand): self.path_template = path_template notifier = self.setup_notifier() + # this thread is done using the database; all future access is done in + # the spawned read_repo() processes, so close the otherwise completely + # idle connection. + connection.close() + logger.info('Entering notifier loop') notifier.loop() @@ -61,14 +66,17 @@ class Command(BaseCommand): if hasattr(thread, 'cancel'): thread.cancel() + @transaction.commit_on_success def setup_notifier(self): '''Set up and configure the inotify machinery and logic. This takes the provided or default path_template and builds a list of directories we need to watch for database updates. It then validates and passes these on to the various pyinotify pieces as necessary and finally builds and returns a notifier object.''' + transaction.commit_manually() arches = Arch.objects.filter(agnostic=False) repos = Repo.objects.all() + transaction.set_dirty() arch_path_map = {arch: None for arch in arches} all_paths = set() total_paths = 0 @@ -91,11 +99,6 @@ class Command(BaseCommand): raise CommandError('path template did not uniquely ' 'determine architecture for each file') - # this thread is done using the database; all future access is done in - # the spawned read_repo() processes, so close the otherwise completely - # idle connection. - connection.close() - # A proper atomic replacement of the database as done by rsync is type # IN_MOVED_TO. repo-add/remove will finish with a IN_CLOSE_WRITE. mask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO -- cgit v1.2.3-2-g168b