Merge branch 'master' of git://projects.archlinux.org/archweb

Conflicts:
	templates/packages/packages_list.html
	templates/public/index.html

1 files changed, 74 insertions, 27 deletions

diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py
index 3f1e9ddf..e26bb800 100644
--- a/devel/management/commands/reporead.py
+++ b/devel/management/commands/reporead.py
@@ -35,12 +35,11 @@ try:
 except ImportError:
     pass
 
-from logging import ERROR, WARNING, INFO, DEBUG
-
 from main.models import Arch, Package, PackageDepend, PackageFile, Repo
+from packages.models import Conflict, Provision, Replacement
 
 logging.basicConfig(
-    level=WARNING,
+    level=logging.WARNING,
     format='%(asctime)s -> %(levelname)s: %(message)s',
     datefmt='%Y-%m-%d %H:%M:%S',
     stream=sys.stderr)
@@ -69,11 +68,11 @@ class Command(BaseCommand):
 
         v = int(options.get('verbosity', 0))
         if v == 0:
-            logger.level = ERROR
+            logger.level = logging.ERROR
         elif v == 1:
-            logger.level = INFO
+            logger.level = logging.INFO
         elif v == 2:
-            logger.level = DEBUG
+            logger.level = logging.DEBUG
 
         import signal, traceback
         handler = lambda sig, stack: traceback.print_stack(stack)
@@ -88,6 +87,9 @@ class Pkg(object):
     bare = ( 'name', 'base', 'arch', 'desc', 'filename',
             'md5sum', 'url', 'builddate', 'packager' )
     number = ( 'csize', 'isize' )
+    collections = ( 'depends', 'optdepends', 'conflicts',
+            'provides', 'replaces', 'groups', 'license', 'files' )
+
     version_re = re.compile(r'^((\d+):)?(.+)-([^-]+)$')
 
     def __init__(self, repo):
@@ -97,6 +99,11 @@ class Pkg(object):
         self.epoch = 0
         for k in self.bare + self.number:
             setattr(self, k, None)
+        for k in self.collections:
+            setattr(self, k, ())
+        # So we can tell the diffence between a package with no files, and a DB
+        # without files entries
+        self.has_files = False
 
     def populate(self, values):
         for k, v in values.iteritems():
@@ -105,16 +112,17 @@ class Pkg(object):
                 setattr(self, k, v[0][:254])
             elif k in self.number:
                 setattr(self, k, long(v[0]))
-            elif k == 'force':
-                setattr(self, k, True)
             elif k == 'version':
                 match = self.version_re.match(v[0])
                 self.ver = match.group(3)
                 self.rel = match.group(4)
                 if match.group(2):
                     self.epoch = int(match.group(2))
+            elif k == 'files':
+                self.files = v
+                self.has_files = True
             else:
-                # files, depends, etc.
+                # anything left in collections
                 setattr(self, k, v)
 
     @property
@@ -174,20 +182,58 @@ def find_user(userstring):
 # lookup more than strictly necessary.
 find_user.cache = {}
 
+DEPEND_RE = re.compile(r"^(.+?)((>=|<=|=|>|<)(.*))?$")
+
 def create_depend(package, dep_str, optional=False):
     depend = PackageDepend(pkg=package, optional=optional)
     # lop off any description first
     parts = dep_str.split(':', 1)
     if len(parts) > 1:
         depend.description = parts[1].strip()
-    match = re.match(r"^(.+?)((>=|<=|=|>|<)(.*))?$", parts[0].strip())
+    match = DEPEND_RE.match(parts[0].strip())
     if match:
         depend.depname = match.group(1)
         if match.group(2):
             depend.depvcmp = match.group(2)
+    else:
+        logger.warning('Package %s had unparsable depend string %s',
+                package.pkgname, dep_str)
+        return None
     depend.save(force_insert=True)
     return depend
 
+def create_related(model, package, rel_str, equals_only=False):
+    related = model(pkg=package)
+    match = DEPEND_RE.match(rel_str)
+    if match:
+        related.name = match.group(1)
+        if match.group(3):
+            comp = match.group(3)
+            if not equals_only:
+                related.comparison = comp
+            elif comp != '=':
+                logger.warning(
+                        'Package %s had unexpected comparison operator %s for %s in %s',
+                        package.pkgname, comp, model.__name__, rel_str)
+        if match.group(4):
+            related.version = match.group(4)
+    else:
+        logger.warning('Package %s had unparsable %s string %s',
+                package.pkgname, model.___name__, rel_str)
+        return None
+    related.save(force_insert=True)
+    return related
+
+def create_multivalued(dbpkg, repopkg, db_attr, repo_attr):
+    '''Populate the simplest of multivalued attributes. These are those that
+    only deal with a 'name' attribute, such as licenses, groups, etc. The input
+    and output objects and attribute names are specified, and everything is
+    done via getattr().'''
+    collection = getattr(dbpkg, db_attr)
+    collection.all().delete()
+    for name in getattr(repopkg, repo_attr):
+        collection.create(name=name)
+
 def populate_pkg(dbpkg, repopkg, force=False, timestamp=None):
     if repopkg.base:
         dbpkg.pkgbase = repopkg.base
@@ -222,22 +268,23 @@ def populate_pkg(dbpkg, repopkg, force=False, timestamp=None):
     populate_files(dbpkg, repopkg, force=force)
 
     dbpkg.packagedepend_set.all().delete()
-    if hasattr(repopkg, 'depends'):
-        for y in repopkg.depends:
-            dep = create_depend(dbpkg, y)
-    if hasattr(repopkg, 'optdepends'):
-        for y in repopkg.optdepends:
-            dep = create_depend(dbpkg, y, True)
+    for y in repopkg.depends:
+        create_depend(dbpkg, y)
+    for y in repopkg.optdepends:
+        create_depend(dbpkg, y, True)
 
-    dbpkg.groups.all().delete()
-    if hasattr(repopkg, 'groups'):
-        for y in repopkg.groups:
-            dbpkg.groups.create(name=y)
+    dbpkg.conflicts.all().delete()
+    for y in repopkg.conflicts:
+        create_related(Conflict, dbpkg, y)
+    dbpkg.provides.all().delete()
+    for y in repopkg.provides:
+        create_related(Provision, dbpkg, y, equals_only=True)
+    dbpkg.replaces.all().delete()
+    for y in repopkg.replaces:
+        create_related(Replacement, dbpkg, y)
 
-    dbpkg.licenses.all().delete()
-    if hasattr(repopkg, 'license'):
-        for y in repopkg.license:
-            dbpkg.licenses.create(name=y)
+    create_multivalued(dbpkg, repopkg, 'groups', 'groups')
+    create_multivalued(dbpkg, repopkg, 'licenses', 'license')
 
 
 def populate_files(dbpkg, repopkg, force=False):
@@ -253,7 +300,7 @@ def populate_files(dbpkg, repopkg, force=False):
         elif dbpkg.files_last_update > dbpkg.last_update:
             return
     # only delete files if we are reading a DB that contains them
-    if hasattr(repopkg, 'files'):
+    if repopkg.has_files:
         dbpkg.packagefile_set.all().delete()
         logger.info("adding %d files for package %s",
                 len(repopkg.files), dbpkg.pkgname)
@@ -267,7 +314,7 @@ def populate_files(dbpkg, repopkg, force=False):
                     is_directory=(filename is None),
                     directory=dirname + '/',
                     filename=filename)
-            pkgfile.save()
+            pkgfile.save(force_insert=True)
         dbpkg.files_last_update = datetime.now()
         dbpkg.save()
 
@@ -316,7 +363,7 @@ def db_update(archname, reponame, pkgs, options):
         pass
     elif not filesonly and \
             len(dbset) > 20 and dbpercent < 50.0 and \
-            not repository.testing:
+            not repository.testing and not repository.staging:
         logger.error(msg)
         raise Exception(msg)
     elif dbpercent < 75.0: