summaryrefslogtreecommitdiff
path: root/devel/management
diff options
context:
space:
mode:
Diffstat (limited to 'devel/management')
-rw-r--r--devel/management/commands/generate_keyring.py6
-rw-r--r--devel/management/commands/import_signatures.py123
-rw-r--r--devel/management/commands/pgp_import.py241
-rw-r--r--devel/management/commands/rematch_developers.py63
-rw-r--r--devel/management/commands/reporead.py102
-rw-r--r--[-rwxr-xr-x]devel/management/commands/reporead_inotify.py19
6 files changed, 353 insertions, 201 deletions
diff --git a/devel/management/commands/generate_keyring.py b/devel/management/commands/generate_keyring.py
index b9117c84..34bcd2f8 100644
--- a/devel/management/commands/generate_keyring.py
+++ b/devel/management/commands/generate_keyring.py
@@ -48,13 +48,17 @@ def generate_keyring(keyserver, keyring):
logger.info("getting all known key IDs")
# Screw you Django, for not letting one natively do value != <empty string>
- key_ids = UserProfile.objects.filter(user__is_active=True,
+ key_ids = UserProfile.objects.filter(
pgp_key__isnull=False).extra(where=["pgp_key != ''"]).values_list(
"pgp_key", flat=True)
logger.info("%d keys fetched from user profiles", len(key_ids))
master_key_ids = MasterKey.objects.values_list("pgp_key", flat=True)
logger.info("%d keys fetched from master keys", len(master_key_ids))
+ # GPG is stupid and interprets any filename without path portion as being
+ # in ~/.gnupg/. Fake it out if we just get a bare filename.
+ if '/' not in keyring:
+ keyring = './%s' % keyring
gpg_cmd = ["gpg", "--no-default-keyring", "--keyring", keyring,
"--keyserver", keyserver, "--recv-keys"]
logger.info("running command: %r", gpg_cmd)
diff --git a/devel/management/commands/import_signatures.py b/devel/management/commands/import_signatures.py
deleted file mode 100644
index ce1aba90..00000000
--- a/devel/management/commands/import_signatures.py
+++ /dev/null
@@ -1,123 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-import_signatures command
-
-Import signatures from a given GPG keyring.
-
-Usage: ./manage.py generate_keyring <keyring_path>
-"""
-
-from collections import namedtuple
-from datetime import datetime
-import logging
-import subprocess
-import sys
-
-from django.core.management.base import BaseCommand, CommandError
-from django.db import transaction
-
-from devel.models import PGPSignature
-
-logging.basicConfig(
- level=logging.INFO,
- format='%(asctime)s -> %(levelname)s: %(message)s',
- datefmt='%Y-%m-%d %H:%M:%S',
- stream=sys.stderr)
-logger = logging.getLogger()
-
-class Command(BaseCommand):
- args = "<keyring_path>"
- help = "Import signatures from a given GPG keyring."
-
- def handle(self, *args, **options):
- v = int(options.get('verbosity', None))
- if v == 0:
- logger.level = logging.ERROR
- elif v == 1:
- logger.level = logging.INFO
- elif v == 2:
- logger.level = logging.DEBUG
-
- if len(args) < 1:
- raise CommandError("keyring_path must be provided")
-
- import_signatures(args[0])
-
-
-SignatureData = namedtuple('SignatureData',
- ('signer', 'signee', 'created', 'expires', 'valid'))
-
-
-def get_date(epoch_string):
- '''Convert a epoch string into a python 'date' object (not datetime).'''
- return datetime.utcfromtimestamp(int(epoch_string)).date()
-
-
-def parse_sigdata(data):
- nodes = {}
- edges = []
- current_pubkey = None
-
- # parse all of the output from our successful GPG command
- logger.info("parsing command output")
- for line in data.split('\n'):
- parts = line.split(':')
- if parts[0] == 'pub':
- current_pubkey = parts[4]
- nodes[current_pubkey] = None
- if parts[0] == 'uid':
- uid = parts[9]
- # only set uid if this is the first one encountered
- if nodes[current_pubkey] is None:
- nodes[current_pubkey] = uid
- if parts[0] == 'sig':
- signer = parts[4]
- created = get_date(parts[5])
- expires = None
- if parts[6]:
- expires = get_date(parts[6])
- valid = parts[1] != '-'
- edge = SignatureData(signer, current_pubkey,
- created, expires, valid)
- edges.append(edge)
-
- return nodes, edges
-
-
-def import_signatures(keyring):
- gpg_cmd = ["gpg", "--no-default-keyring", "--keyring", keyring,
- "--list-sigs", "--with-colons", "--fixed-list-mode"]
- logger.info("running command: %r", gpg_cmd)
- proc = subprocess.Popen(gpg_cmd, stdout=subprocess.PIPE)
- outdata, errdata = proc.communicate()
- if proc.returncode != 0:
- logger.error(errdata)
- raise subprocess.CalledProcessError(proc.returncode, gpg_cmd)
-
- nodes, edges = parse_sigdata(outdata)
-
- # now prune the data down to what we actually want.
- # prune edges not in nodes, remove duplicates, and self-sigs
- pruned_edges = set(edge for edge in edges
- if edge.signer in nodes and edge.signer != edge.signee)
-
- logger.info("creating or finding %d signatures", len(pruned_edges))
- created_ct = updated_ct = 0
- with transaction.commit_on_success():
- for edge in pruned_edges:
- sig, created = PGPSignature.objects.get_or_create(
- signer=edge.signer, signee=edge.signee,
- created=edge.created, expires=edge.expires,
- defaults={ 'valid': edge.valid })
- if sig.valid != edge.valid:
- sig.valid = edge.valid
- sig.save()
- updated_ct = 1
- if created:
- created_ct += 1
-
- sig_ct = PGPSignature.objects.all().count()
- logger.info("%d total signatures in database", sig_ct)
- logger.info("created %d, updated %d signatures", created_ct, updated_ct)
-
-# vim: set ts=4 sw=4 et:
diff --git a/devel/management/commands/pgp_import.py b/devel/management/commands/pgp_import.py
new file mode 100644
index 00000000..10e6cfcb
--- /dev/null
+++ b/devel/management/commands/pgp_import.py
@@ -0,0 +1,241 @@
+# -*- coding: utf-8 -*-
+"""
+pgp_import command
+
+Import keys and signatures from a given GPG keyring.
+
+Usage: ./manage.py pgp_import <keyring_path>
+"""
+
+from collections import namedtuple, OrderedDict
+from datetime import datetime
+import logging
+from pytz import utc
+import subprocess
+import sys
+
+from django.core.management.base import BaseCommand, CommandError
+from django.db import transaction
+
+from devel.models import DeveloperKey, PGPSignature
+from devel.utils import UserFinder
+
+
+logging.basicConfig(
+ level=logging.INFO,
+ format='%(asctime)s -> %(levelname)s: %(message)s',
+ datefmt='%Y-%m-%d %H:%M:%S',
+ stream=sys.stderr)
+logger = logging.getLogger()
+
+class Command(BaseCommand):
+ args = "<keyring_path>"
+ help = "Import keys and signatures from a given GPG keyring."
+
+ def handle(self, *args, **options):
+ v = int(options.get('verbosity', None))
+ if v == 0:
+ logger.level = logging.ERROR
+ elif v == 1:
+ logger.level = logging.INFO
+ elif v == 2:
+ logger.level = logging.DEBUG
+
+ if len(args) < 1:
+ raise CommandError("keyring_path must be provided")
+
+ import_keys(args[0])
+ import_signatures(args[0])
+
+
+def get_date(epoch_string):
+ '''Convert a epoch string into a python 'date' object (not datetime).'''
+ if not epoch_string:
+ return None
+ return datetime.utcfromtimestamp(int(epoch_string)).date()
+
+
+def get_datetime(epoch_string):
+ '''Convert a epoch string into a python 'datetime' object.'''
+ if not epoch_string:
+ return None
+ return datetime.utcfromtimestamp(int(epoch_string)).replace(tzinfo=utc)
+
+
+def call_gpg(keyring, *args):
+ # GPG is stupid and interprets any filename without path portion as being
+ # in ~/.gnupg/. Fake it out if we just get a bare filename.
+ if '/' not in keyring:
+ keyring = './%s' % keyring
+ gpg_cmd = ["gpg2", "--no-default-keyring", "--keyring", keyring,
+ "--with-colons", "--fixed-list-mode"]
+ gpg_cmd.extend(args)
+ logger.info("running command: %s", ' '.join(gpg_cmd))
+ proc = subprocess.Popen(gpg_cmd, stdout=subprocess.PIPE)
+ outdata, errdata = proc.communicate()
+ if proc.returncode != 0:
+ logger.error(errdata)
+ raise subprocess.CalledProcessError(proc.returncode, gpg_cmd)
+ return outdata
+
+
+class KeyData(object):
+ def __init__(self, key, created, expires):
+ self.key = key
+ self.created = get_datetime(created)
+ self.expires = get_datetime(expires)
+ self.parent = None
+ self.revoked = None
+ self.db_id = None
+
+
+def parse_keydata(data):
+ keys = OrderedDict()
+ current_pubkey = None
+
+ # parse all of the output from our successful GPG command
+ logger.info("parsing command output")
+ for line in data.split('\n'):
+ parts = line.split(':')
+ if parts[0] == 'pub':
+ key = parts[4]
+ current_pubkey = key
+ keys[key] = KeyData(key, parts[5], parts[6])
+ node = parts[0]
+ elif parts[0] == 'sub':
+ key = parts[4]
+ keys[key] = KeyData(key, parts[5], parts[6])
+ keys[key].parent = current_pubkey
+ node = parts[0]
+ elif parts[0] == 'uid':
+ node = parts[0]
+ elif parts[0] == 'rev' and node in ('pub', 'sub'):
+ keys[current_pubkey].revoked = get_datetime(parts[5])
+
+ return keys
+
+
+def find_key_owner(key, keys, finder):
+ '''Recurse up the chain, looking for an owner.'''
+ if key is None:
+ return None
+ owner = finder.find_by_pgp_key(key.key)
+ if owner:
+ return owner
+ if key.parent:
+ return find_key_owner(keys[key.parent], keys, finder)
+ return None
+
+
+def import_keys(keyring):
+ outdata = call_gpg(keyring, "--list-sigs")
+ keydata = parse_keydata(outdata)
+
+ logger.info("creating or finding %d keys", len(keydata))
+ created_ct = updated_ct = 0
+ with transaction.commit_on_success():
+ finder = UserFinder()
+ # we are dependent on parents coming before children; parse_keydata
+ # uses an OrderedDict to ensure this is the case.
+ for data in keydata.values():
+ parent_id = None
+ if data.parent:
+ parent_data = keydata.get(data.parent, None)
+ if parent_data:
+ parent_id = parent_data.db_id
+ other = {
+ 'expires': data.expires,
+ 'revoked': data.revoked,
+ 'parent_id': parent_id,
+ }
+ dkey, created = DeveloperKey.objects.get_or_create(
+ key=data.key, created=data.created, defaults=other)
+ data.db_id = dkey.id
+
+ # set or update any additional data we might need to
+ needs_save = False
+ if created:
+ created_ct += 1
+ else:
+ for k, v in other.items():
+ if getattr(dkey, k) != v:
+ setattr(dkey, k, v)
+ needs_save = True
+ if dkey.owner_id is None:
+ owner = find_key_owner(data, keydata, finder)
+ if owner is not None:
+ dkey.owner = owner
+ needs_save = True
+ if needs_save:
+ dkey.save()
+ updated_ct += 1
+
+ key_ct = DeveloperKey.objects.all().count()
+ logger.info("%d total keys in database", key_ct)
+ logger.info("created %d, updated %d keys", created_ct, updated_ct)
+
+
+SignatureData = namedtuple('SignatureData',
+ ('signer', 'signee', 'created', 'expires', 'valid'))
+
+
+def parse_sigdata(data):
+ nodes = {}
+ edges = []
+ current_pubkey = None
+
+ # parse all of the output from our successful GPG command
+ logger.info("parsing command output")
+ for line in data.split('\n'):
+ parts = line.split(':')
+ if parts[0] == 'pub':
+ current_pubkey = parts[4]
+ nodes[current_pubkey] = None
+ if parts[0] == 'uid':
+ uid = parts[9]
+ # only set uid if this is the first one encountered
+ if nodes[current_pubkey] is None:
+ nodes[current_pubkey] = uid
+ if parts[0] == 'sig':
+ signer = parts[4]
+ created = get_date(parts[5])
+ expires = None
+ if parts[6]:
+ expires = get_date(parts[6])
+ valid = parts[1] != '-'
+ edge = SignatureData(signer, current_pubkey,
+ created, expires, valid)
+ edges.append(edge)
+
+ return nodes, edges
+
+
+def import_signatures(keyring):
+ outdata = call_gpg(keyring, "--list-sigs")
+ nodes, edges = parse_sigdata(outdata)
+
+ # now prune the data down to what we actually want.
+ # prune edges not in nodes, remove duplicates, and self-sigs
+ pruned_edges = {edge for edge in edges
+ if edge.signer in nodes and edge.signer != edge.signee}
+
+ logger.info("creating or finding %d signatures", len(pruned_edges))
+ created_ct = updated_ct = 0
+ with transaction.commit_on_success():
+ for edge in pruned_edges:
+ sig, created = PGPSignature.objects.get_or_create(
+ signer=edge.signer, signee=edge.signee,
+ created=edge.created, expires=edge.expires,
+ defaults={ 'valid': edge.valid })
+ if sig.valid != edge.valid:
+ sig.valid = edge.valid
+ sig.save()
+ updated_ct = 1
+ if created:
+ created_ct += 1
+
+ sig_ct = PGPSignature.objects.all().count()
+ logger.info("%d total signatures in database", sig_ct)
+ logger.info("created %d, updated %d signatures", created_ct, updated_ct)
+
+# vim: set ts=4 sw=4 et:
diff --git a/devel/management/commands/rematch_developers.py b/devel/management/commands/rematch_developers.py
index 8383cc8d..2b379588 100644
--- a/devel/management/commands/rematch_developers.py
+++ b/devel/management/commands/rematch_developers.py
@@ -46,52 +46,53 @@ class Command(NoArgsCommand):
@transaction.commit_on_success
def match_packager(finder):
- logger.info("getting all unmatched packages")
+ logger.info("getting all unmatched packager strings")
package_count = matched_count = 0
- unknown = set()
-
- for package in Package.objects.filter(packager__isnull=True):
- if package.packager_str in unknown:
- continue
- logger.debug("package %s, packager string %s",
- package.pkgname, package.packager_str)
- package_count += 1
- user = finder.find(package.packager_str)
+ mapping = {}
+
+ unmatched = Package.objects.filter(packager__isnull=True).values_list(
+ 'packager_str', flat=True).order_by().distinct()
+
+ logger.info("%d packager strings retrieved", len(unmatched))
+ for packager in unmatched:
+ logger.debug("packager string %s", packager)
+ user = finder.find(packager)
if user:
- package.packager = user
+ mapping[packager] = user
logger.debug(" found user %s" % user.username)
- package.save()
matched_count += 1
- else:
- unknown.add(package.packager_str)
- logger.info("%d packager strings checked, %d newly matched",
+ for packager_str, user in mapping.items():
+ package_count += Package.objects.filter(packager__isnull=True,
+ packager_str=packager_str).update(packager=user)
+
+ logger.info("%d packages updated, %d packager strings matched",
package_count, matched_count)
- logger.debug("unknown packagers:\n%s",
- "\n".join(unknown))
@transaction.commit_on_success
def match_flagrequest(finder):
- logger.info("getting all non-user flag requests")
+ logger.info("getting all flag request email addresses from unknown users")
req_count = matched_count = 0
- unknown = set()
-
- for request in FlagRequest.objects.filter(user__isnull=True):
- if request.user_email in unknown:
- continue
- logger.debug("email %s", request.user_email)
- req_count += 1
- user = finder.find_by_email(request.user_email)
+ mapping = {}
+
+ unmatched = FlagRequest.objects.filter(user__isnull=True).values_list(
+ 'user_email', flat=True).order_by().distinct()
+
+ logger.info("%d email addresses retrieved", len(unmatched))
+ for user_email in unmatched:
+ logger.debug("email %s", user_email)
+ user = finder.find_by_email(user_email)
if user:
- request.user = user
+ mapping[user_email] = user
logger.debug(" found user %s" % user.username)
- request.save()
matched_count += 1
- else:
- unknown.add(request.user_email)
- logger.info("%d request emails checked, %d newly matched",
+ for user_email, user in mapping.items():
+ req_count += FlagRequest.objects.filter(user__isnull=True,
+ user_email=user_email).update(user=user)
+
+ logger.info("%d request emails updated, %d emails matched",
req_count, matched_count)
# vim: set ts=4 sw=4 et:
diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index cf98f004..1e456c8c 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -14,6 +14,7 @@ Example:
"""
from collections import defaultdict
+from copy import copy
import io
import os
import re
@@ -27,11 +28,12 @@ from pytz import utc
from django.core.management.base import BaseCommand, CommandError
from django.db import connections, router, transaction
from django.db.utils import IntegrityError
+from django.utils.timezone import now
from devel.utils import UserFinder
-from main.models import Arch, Package, PackageDepend, PackageFile, Repo
-from main.utils import utc_now
-from packages.models import Conflict, Provision, Replacement
+from main.models import Arch, Package, PackageFile, Repo
+from packages.models import Depend, Conflict, Provision, Replacement, Update
+from packages.utils import parse_version
logging.basicConfig(
@@ -78,10 +80,9 @@ class RepoPackage(object):
bare = ( 'name', 'base', 'arch', 'filename',
'md5sum', 'sha256sum', 'url', 'packager' )
number = ( 'csize', 'isize' )
- collections = ( 'depends', 'optdepends', 'conflicts',
- 'provides', 'replaces', 'groups', 'license', 'files' )
-
- version_re = re.compile(r'^((\d+):)?(.+)-([^-]+)$')
+ collections = ( 'depends', 'optdepends', 'makedepends', 'checkdepends',
+ 'conflicts', 'provides', 'replaces', 'groups', 'license',
+ 'files' )
def __init__(self, repo):
self.repo = repo
@@ -111,11 +112,7 @@ class RepoPackage(object):
v[0] = 'missing'
setattr(self, k, v[0])
elif k == 'version':
- match = self.version_re.match(v[0])
- self.ver = match.group(3)
- self.rel = match.group(4)
- if match.group(2):
- self.epoch = int(match.group(2))
+ self.ver, self.rel, self.epoch = parse_version(v[0])
elif k == 'builddate':
try:
builddate = datetime.utcfromtimestamp(int(v[0]))
@@ -143,19 +140,21 @@ class RepoPackage(object):
return u'%s-%s' % (self.ver, self.rel)
-DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.*))?$")
+DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.+))?$")
-def create_depend(package, dep_str, optional=False):
- depend = PackageDepend(pkg=package, optional=optional)
+def create_depend(package, dep_str, deptype='D'):
+ depend = Depend(pkg=package, deptype=deptype)
# lop off any description first
parts = dep_str.split(':', 1)
if len(parts) > 1:
depend.description = parts[1].strip()
match = DEPEND_RE.match(parts[0].strip())
if match:
- depend.depname = match.group(1)
- if match.group(2):
- depend.depvcmp = match.group(2)
+ depend.name = match.group(1)
+ if match.group(3):
+ depend.comparison = match.group(3)
+ if match.group(4):
+ depend.version = match.group(4)
else:
logger.warning('Package %s had unparsable depend string %s',
package.pkgname, dep_str)
@@ -183,6 +182,7 @@ def create_related(model, package, rel_str, equals_only=False):
return None
return related
+
def create_multivalued(dbpkg, repopkg, db_attr, repo_attr):
'''Populate the simplest of multivalued attributes. These are those that
only deal with a 'name' attribute, such as licenses, groups, etc. The input
@@ -236,8 +236,10 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None):
dbpkg.depends.all().delete()
deps = [create_depend(dbpkg, y) for y in repopkg.depends]
- deps += [create_depend(dbpkg, y, True) for y in repopkg.optdepends]
- PackageDepend.objects.bulk_create(deps)
+ deps += [create_depend(dbpkg, y, 'O') for y in repopkg.optdepends]
+ deps += [create_depend(dbpkg, y, 'M') for y in repopkg.makedepends]
+ deps += [create_depend(dbpkg, y, 'C') for y in repopkg.checkdepends]
+ Depend.objects.bulk_create(deps)
dbpkg.conflicts.all().delete()
conflicts = [create_related(Conflict, dbpkg, y) for y in repopkg.conflicts]
@@ -275,7 +277,7 @@ def populate_files(dbpkg, repopkg, force=False):
return
if not dbpkg.files_last_update or not dbpkg.last_update:
pass
- elif dbpkg.files_last_update > dbpkg.last_update:
+ elif dbpkg.files_last_update >= dbpkg.last_update:
return
# only delete files if we are reading a DB that contains them
@@ -284,17 +286,24 @@ def populate_files(dbpkg, repopkg, force=False):
logger.info("adding %d files for package %s",
len(repopkg.files), dbpkg.pkgname)
pkg_files = []
- for f in repopkg.files:
- dirname, filename = f.rsplit('/', 1)
+ # sort in normal alpha-order that pacman uses, rather than makepkg's
+ # default breadth-first, directory-first ordering
+ files = sorted(repopkg.files)
+ for f in files:
+ if '/' in f:
+ dirname, filename = f.rsplit('/', 1)
+ dirname += '/'
+ else:
+ dirname, filename = '', f
if filename == '':
filename = None
pkgfile = PackageFile(pkg=dbpkg,
is_directory=(filename is None),
- directory=dirname + '/',
+ directory=dirname,
filename=filename)
pkg_files.append(pkgfile)
PackageFile.objects.bulk_create(pkg_files)
- dbpkg.files_last_update = utc_now()
+ dbpkg.files_last_update = now()
dbpkg.save()
@@ -307,7 +316,7 @@ def update_common(archname, reponame, pkgs, sanity_check=True):
transaction.set_dirty()
repository = Repo.objects.get(name__iexact=reponame)
- architecture = Arch.objects.get(name__iexact=archname)
+ architecture = Arch.objects.get(name=archname)
# no-arg order_by() removes even the default ordering; we don't need it
dbpkgs = Package.objects.filter(
arch=architecture, repo=repository).order_by()
@@ -346,35 +355,42 @@ def db_update(archname, reponame, pkgs, force=False):
logger.info('Updating %s (%s)', reponame, archname)
dbpkgs = update_common(archname, reponame, pkgs, True)
repository = Repo.objects.get(name__iexact=reponame)
- architecture = Arch.objects.get(name__iexact=archname)
+ architecture = Arch.objects.get(name=archname)
# This makes our inner loop where we find packages by name *way* more
# efficient by not having to go to the database for each package to
# SELECT them by name.
- dbdict = dict((dbpkg.pkgname, dbpkg) for dbpkg in dbpkgs)
+ dbdict = {dbpkg.pkgname: dbpkg for dbpkg in dbpkgs}
dbset = set(dbdict.keys())
- syncset = set([pkg.name for pkg in pkgs])
+ syncset = {pkg.name for pkg in pkgs}
in_sync_not_db = syncset - dbset
logger.info("%d packages in sync not db", len(in_sync_not_db))
# packages in syncdb and not in database (add to database)
for pkg in (pkg for pkg in pkgs if pkg.name in in_sync_not_db):
logger.info("Adding package %s", pkg.name)
- dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository)
+ timestamp = now()
+ dbpkg = Package(pkgname=pkg.name, arch=architecture, repo=repository,
+ created=timestamp)
try:
with transaction.commit_on_success():
- populate_pkg(dbpkg, pkg, timestamp=utc_now())
+ populate_pkg(dbpkg, pkg, timestamp=timestamp)
+ Update.objects.log_update(None, dbpkg)
except IntegrityError:
- logger.warning("Could not add package %s; "
- "not fatal if another thread beat us to it.",
- pkg.name, exc_info=True)
+ if architecture.agnostic:
+ logger.warning("Could not add package %s; "
+ "not fatal if another thread beat us to it.",
+ pkg.name)
+ else:
+ logger.exception("Could not add package %s", pkg.name)
# packages in database and not in syncdb (remove from database)
for pkgname in (dbset - syncset):
logger.info("Removing package %s", pkgname)
dbpkg = dbdict[pkgname]
with transaction.commit_on_success():
+ Update.objects.log_update(dbpkg, None)
# no race condition here as long as simultaneous threads both
# issue deletes; second delete will be a no-op
delete_pkg_files(dbpkg)
@@ -391,7 +407,7 @@ def db_update(archname, reponame, pkgs, force=False):
if not force and pkg_same_version(pkg, dbpkg):
continue
elif not force:
- timestamp = utc_now()
+ timestamp = now()
# The odd select_for_update song and dance here are to ensure
# simultaneous updates don't happen on a package, causing
@@ -402,7 +418,9 @@ def db_update(archname, reponame, pkgs, force=False):
logger.debug("Package %s was already updated", pkg.name)
continue
logger.info("Updating package %s", pkg.name)
+ prevpkg = copy(dbpkg)
populate_pkg(dbpkg, pkg, force=force, timestamp=timestamp)
+ Update.objects.log_update(prevpkg, dbpkg)
logger.info('Finished updating arch: %s', archname)
@@ -413,7 +431,7 @@ def filesonly_update(archname, reponame, pkgs, force=False):
"""
logger.info('Updating files for %s (%s)', reponame, archname)
dbpkgs = update_common(archname, reponame, pkgs, False)
- dbdict = dict((dbpkg.pkgname, dbpkg) for dbpkg in dbpkgs)
+ dbdict = {dbpkg.pkgname: dbpkg for dbpkg in dbpkgs}
dbset = set(dbdict.keys())
for pkg in (pkg for pkg in pkgs if pkg.name in dbset):
@@ -425,7 +443,7 @@ def filesonly_update(archname, reponame, pkgs, force=False):
with transaction.commit_on_success():
if not dbpkg.files_last_update or not dbpkg.last_update:
pass
- elif not force and dbpkg.files_last_update > dbpkg.last_update:
+ elif not force and dbpkg.files_last_update >= dbpkg.last_update:
logger.debug("Files for %s are up to date", pkg.name)
continue
dbpkg = Package.objects.select_for_update().get(id=dbpkg.id)
@@ -509,7 +527,7 @@ def locate_arch(arch):
if isinstance(arch, Arch):
return arch
try:
- return Arch.objects.get(name__iexact=arch)
+ return Arch.objects.get(name=arch)
except Arch.DoesNotExist:
raise CommandError(
'Specified architecture %s is not currently known.' % arch)
@@ -541,6 +559,12 @@ def read_repo(primary_arch, repo_file, options):
package.name, repo_file, package.arch))
del packages
+ database = router.db_for_write(Package)
+ connection = connections[database]
+ if connection.vendor == 'sqlite':
+ cursor = connection.cursor()
+ cursor.execute('PRAGMA synchronous = NORMAL')
+
logger.info('Starting database updates for %s.', repo_file)
for arch in sorted(packages_arches.keys()):
if filesonly:
@@ -548,6 +572,8 @@ def read_repo(primary_arch, repo_file, options):
else:
db_update(arch, repo, packages_arches[arch], force)
logger.info('Finished database updates for %s.', repo_file)
+ connection.commit()
+ connection.close()
return 0
# vim: set ts=4 sw=4 et:
diff --git a/devel/management/commands/reporead_inotify.py b/devel/management/commands/reporead_inotify.py
index c74762eb..8c1e47bf 100755..100644
--- a/devel/management/commands/reporead_inotify.py
+++ b/devel/management/commands/reporead_inotify.py
@@ -23,7 +23,7 @@ import threading
import time
from django.core.management.base import BaseCommand, CommandError
-from django.db import connection
+from django.db import connection, transaction
from main.models import Arch, Repo
from .reporead import read_repo
@@ -53,6 +53,11 @@ class Command(BaseCommand):
self.path_template = path_template
notifier = self.setup_notifier()
+ # this thread is done using the database; all future access is done in
+ # the spawned read_repo() processes, so close the otherwise completely
+ # idle connection.
+ connection.close()
+
logger.info('Entering notifier loop')
notifier.loop()
@@ -61,15 +66,18 @@ class Command(BaseCommand):
if hasattr(thread, 'cancel'):
thread.cancel()
+ @transaction.commit_on_success
def setup_notifier(self):
'''Set up and configure the inotify machinery and logic.
This takes the provided or default path_template and builds a list of
directories we need to watch for database updates. It then validates
and passes these on to the various pyinotify pieces as necessary and
finally builds and returns a notifier object.'''
+ transaction.commit_manually()
arches = Arch.objects.filter(agnostic=False)
repos = Repo.objects.all()
- arch_path_map = dict((arch, None) for arch in arches)
+ transaction.set_dirty()
+ arch_path_map = {arch: None for arch in arches}
all_paths = set()
total_paths = 0
for arch in arches:
@@ -77,7 +85,7 @@ class Command(BaseCommand):
for repo in repos)
# take a python format string and generate all unique combinations
# of directories from it; using set() ensures we filter it down
- paths = set(self.path_template % values for values in combos)
+ paths = {self.path_template % values for values in combos}
total_paths += len(paths)
all_paths |= paths
arch_path_map[arch] = paths
@@ -91,11 +99,6 @@ class Command(BaseCommand):
raise CommandError('path template did not uniquely '
'determine architecture for each file')
- # this thread is done using the database; all future access is done in
- # the spawned read_repo() processes, so close the otherwise completely
- # idle connection.
- connection.close()
-
# A proper atomic replacement of the database as done by rsync is type
# IN_MOVED_TO. repo-add/remove will finish with a IN_CLOSE_WRITE.
mask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_MOVED_TO