From 59b176ed025548118f750e0d506f8a1d20cc4635 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 21 Mar 2011 13:46:09 -0500 Subject: Don't bail on staging repo being short on packages Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 3f1e9ddf..231cad01 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -316,7 +316,7 @@ def db_update(archname, reponame, pkgs, options): pass elif not filesonly and \ len(dbset) > 20 and dbpercent < 50.0 and \ - not repository.testing: + not repository.testing and not repository.staging: logger.error(msg) raise Exception(msg) elif dbpercent < 75.0: -- cgit v1.2.3-2-g168b From 857cae2e96aa0d23b0702d5ad9db572787f657a9 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 16 Feb 2011 18:22:14 -0600 Subject: reporead: refactor multivalued attribute creation This will come in more handy with our new models, but we can adapt groups and licenses to use it first. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 231cad01..0c835555 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -188,6 +188,17 @@ def create_depend(package, dep_str, optional=False): depend.save(force_insert=True) return depend +def create_multivalued(dbpkg, repopkg, db_attr, repo_attr): + '''Populate the simplest of multivalued attributes. These are those that + only deal with a 'name' attribute, such as licenses, groups, etc. The input + and output objects and attribute names are specified, and everything is + done via hasattr()/getattr().''' + collection = getattr(dbpkg, db_attr) + collection.all().delete() + if hasattr(repopkg, repo_attr): + for name in getattr(repopkg, repo_attr): + collection.create(name=name) + def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): if repopkg.base: dbpkg.pkgbase = repopkg.base @@ -229,15 +240,8 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): for y in repopkg.optdepends: dep = create_depend(dbpkg, y, True) - dbpkg.groups.all().delete() - if hasattr(repopkg, 'groups'): - for y in repopkg.groups: - dbpkg.groups.create(name=y) - - dbpkg.licenses.all().delete() - if hasattr(repopkg, 'license'): - for y in repopkg.license: - dbpkg.licenses.create(name=y) + create_multivalued(dbpkg, repopkg, 'groups', 'groups') + create_multivalued(dbpkg, repopkg, 'licenses', 'license') def populate_files(dbpkg, repopkg, force=False): -- cgit v1.2.3-2-g168b From 151eafb0192957c489fa5b8010c081eb6db5f925 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 23 Mar 2011 16:43:39 -0500 Subject: Remove need to import each individual logger constant Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 0c835555..8ce6eab8 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -35,12 +35,10 @@ try: except ImportError: pass -from logging import ERROR, WARNING, INFO, DEBUG - from main.models import Arch, Package, PackageDepend, PackageFile, Repo logging.basicConfig( - level=WARNING, + level=logging.WARNING, format='%(asctime)s -> %(levelname)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S', stream=sys.stderr) @@ -69,11 +67,11 @@ class Command(BaseCommand): v = int(options.get('verbosity', 0)) if v == 0: - logger.level = ERROR + logger.level = logging.ERROR elif v == 1: - logger.level = INFO + logger.level = logging.INFO elif v == 2: - logger.level = DEBUG + logger.level = logging.DEBUG import signal, traceback handler = lambda sig, stack: traceback.print_stack(stack) -- cgit v1.2.3-2-g168b From d6b148779f3a1119e30fa75c63a2a0e46938098c Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 23 Mar 2011 16:44:17 -0500 Subject: reporead: read in provisions, conflicts, and replacements Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 45 ++++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 8ce6eab8..708b8a59 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -36,6 +36,7 @@ except ImportError: pass from main.models import Arch, Package, PackageDepend, PackageFile, Repo +from packages.models import Conflict, Provision, Replacement logging.basicConfig( level=logging.WARNING, @@ -172,20 +173,48 @@ def find_user(userstring): # lookup more than strictly necessary. find_user.cache = {} +DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.*))?$") + def create_depend(package, dep_str, optional=False): depend = PackageDepend(pkg=package, optional=optional) # lop off any description first parts = dep_str.split(':', 1) if len(parts) > 1: depend.description = parts[1].strip() - match = re.match(r"^(.+?)((>=|<=|=|>|<)(.*))?$", parts[0].strip()) + match = DEPEND_RE.match(parts[0].strip()) if match: depend.depname = match.group(1) if match.group(2): depend.depvcmp = match.group(2) + else: + logger.warning('Package %s had unparsable depend string %s', + package.pkgname, dep_str) + return None depend.save(force_insert=True) return depend +def create_related(model, package, rel_str, equals_only=False): + related = model(pkg=package) + match = DEPEND_RE.match(rel_str) + if match: + related.name = match.group(1) + if match.group(3): + comp = match.group(3) + if not equals_only: + related.comparison = comp + elif comp != '=': + logger.warning( + 'Package %s had unexpected comparison operator %s for %s in %s', + package.pkgname, comp, model.__name__, rel_str) + if match.group(4): + related.version = match.group(4) + else: + logger.warning('Package %s had unparsable %s string %s', + package.pkgname, model.___name__, rel_str) + return None + related.save(force_insert=True) + return related + def create_multivalued(dbpkg, repopkg, db_attr, repo_attr): '''Populate the simplest of multivalued attributes. These are those that only deal with a 'name' attribute, such as licenses, groups, etc. The input @@ -233,10 +262,20 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): dbpkg.packagedepend_set.all().delete() if hasattr(repopkg, 'depends'): for y in repopkg.depends: - dep = create_depend(dbpkg, y) + create_depend(dbpkg, y) if hasattr(repopkg, 'optdepends'): for y in repopkg.optdepends: - dep = create_depend(dbpkg, y, True) + create_depend(dbpkg, y, True) + + if hasattr(repopkg, 'conflicts'): + for y in repopkg.conflicts: + create_related(Conflict, dbpkg, y) + if hasattr(repopkg, 'provides'): + for y in repopkg.provides: + create_related(Provision, dbpkg, y, equals_only=True) + if hasattr(repopkg, 'replaces'): + for y in repopkg.replaces: + create_related(Replacement, dbpkg, y) create_multivalued(dbpkg, repopkg, 'groups', 'groups') create_multivalued(dbpkg, repopkg, 'licenses', 'license') -- cgit v1.2.3-2-g168b From dad2ca8b3e42cbf0ad5a67be7016426ec4835a19 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 23 Mar 2011 20:19:00 -0500 Subject: Clear out package relation sets before adding new values Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 708b8a59..277196e2 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -267,12 +267,15 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): for y in repopkg.optdepends: create_depend(dbpkg, y, True) + dbpkg.conflicts.all().delete() if hasattr(repopkg, 'conflicts'): for y in repopkg.conflicts: create_related(Conflict, dbpkg, y) + dbpkg.provides.all().delete() if hasattr(repopkg, 'provides'): for y in repopkg.provides: create_related(Provision, dbpkg, y, equals_only=True) + dbpkg.replaces.all().delete() if hasattr(repopkg, 'replaces'): for y in repopkg.replaces: create_related(Replacement, dbpkg, y) -- cgit v1.2.3-2-g168b From ca370499d863dc2ce86fbeb26cf810d3808239f6 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 23 Mar 2011 20:32:55 -0500 Subject: reporead: remove the need for hasattr() checks Ensure all our multivalued attributes already exist on the object beforehand, and add some special sauce to handle the difference between a package without files and a database without files entries. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 51 ++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 24 deletions(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 277196e2..35bb7da3 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -87,6 +87,9 @@ class Pkg(object): bare = ( 'name', 'base', 'arch', 'desc', 'filename', 'md5sum', 'url', 'builddate', 'packager' ) number = ( 'csize', 'isize' ) + collections = ( 'depends', 'optdepends', 'conflicts', + 'replaces', 'groups', 'license', 'files' ) + version_re = re.compile(r'^((\d+):)?(.+)-([^-]+)$') def __init__(self, repo): @@ -96,6 +99,11 @@ class Pkg(object): self.epoch = 0 for k in self.bare + self.number: setattr(self, k, None) + for k in self.collections: + setattr(self, k, ()) + # So we can tell the diffence between a package with no files, and a DB + # without files entries + self.has_files = False def populate(self, values): for k, v in values.iteritems(): @@ -104,16 +112,17 @@ class Pkg(object): setattr(self, k, v[0][:254]) elif k in self.number: setattr(self, k, long(v[0])) - elif k == 'force': - setattr(self, k, True) elif k == 'version': match = self.version_re.match(v[0]) self.ver = match.group(3) self.rel = match.group(4) if match.group(2): self.epoch = int(match.group(2)) + elif k == 'files': + self.files = v + self.has_files = True else: - # files, depends, etc. + # anything left in collections setattr(self, k, v) @property @@ -219,12 +228,11 @@ def create_multivalued(dbpkg, repopkg, db_attr, repo_attr): '''Populate the simplest of multivalued attributes. These are those that only deal with a 'name' attribute, such as licenses, groups, etc. The input and output objects and attribute names are specified, and everything is - done via hasattr()/getattr().''' + done via getattr().''' collection = getattr(dbpkg, db_attr) collection.all().delete() - if hasattr(repopkg, repo_attr): - for name in getattr(repopkg, repo_attr): - collection.create(name=name) + for name in getattr(repopkg, repo_attr): + collection.create(name=name) def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): if repopkg.base: @@ -260,25 +268,20 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): populate_files(dbpkg, repopkg, force=force) dbpkg.packagedepend_set.all().delete() - if hasattr(repopkg, 'depends'): - for y in repopkg.depends: - create_depend(dbpkg, y) - if hasattr(repopkg, 'optdepends'): - for y in repopkg.optdepends: - create_depend(dbpkg, y, True) + for y in repopkg.depends: + create_depend(dbpkg, y) + for y in repopkg.optdepends: + create_depend(dbpkg, y, True) dbpkg.conflicts.all().delete() - if hasattr(repopkg, 'conflicts'): - for y in repopkg.conflicts: - create_related(Conflict, dbpkg, y) + for y in repopkg.conflicts: + create_related(Conflict, dbpkg, y) dbpkg.provides.all().delete() - if hasattr(repopkg, 'provides'): - for y in repopkg.provides: - create_related(Provision, dbpkg, y, equals_only=True) + for y in repopkg.provides: + create_related(Provision, dbpkg, y, equals_only=True) dbpkg.replaces.all().delete() - if hasattr(repopkg, 'replaces'): - for y in repopkg.replaces: - create_related(Replacement, dbpkg, y) + for y in repopkg.replaces: + create_related(Replacement, dbpkg, y) create_multivalued(dbpkg, repopkg, 'groups', 'groups') create_multivalued(dbpkg, repopkg, 'licenses', 'license') @@ -297,7 +300,7 @@ def populate_files(dbpkg, repopkg, force=False): elif dbpkg.files_last_update > dbpkg.last_update: return # only delete files if we are reading a DB that contains them - if hasattr(repopkg, 'files'): + if repopkg.has_files: dbpkg.packagefile_set.all().delete() logger.info("adding %d files for package %s", len(repopkg.files), dbpkg.pkgname) @@ -311,7 +314,7 @@ def populate_files(dbpkg, repopkg, force=False): is_directory=(filename is None), directory=dirname + '/', filename=filename) - pkgfile.save() + pkgfile.save(force_insert=True) dbpkg.files_last_update = datetime.now() dbpkg.save() -- cgit v1.2.3-2-g168b From f6464da0e14047e9db803f4b40c18ba644e75f24 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 24 Mar 2011 00:47:23 -0500 Subject: Add provides to collections list Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'devel') diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 35bb7da3..e26bb800 100644 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -88,7 +88,7 @@ class Pkg(object): 'md5sum', 'url', 'builddate', 'packager' ) number = ( 'csize', 'isize' ) collections = ( 'depends', 'optdepends', 'conflicts', - 'replaces', 'groups', 'license', 'files' ) + 'provides', 'replaces', 'groups', 'license', 'files' ) version_re = re.compile(r'^((\d+):)?(.+)-([^-]+)$') -- cgit v1.2.3-2-g168b