From 909b017c08109bda405a3e38a59cbf26211e6683 Mon Sep 17 00:00:00 2001
From: Xavier Chantry <shiningxc@gmail.com>
Date: Fri, 29 Aug 2008 20:23:26 +0200
Subject: Replace check_archlinux.py by check_archlinux/check_packages.py

The old script had several problems so I decided to do a full rewrite.

The improvements include :

* better and safer parsing of PKGBUILDs
It now uses separate parse_pkgbuilds.sh bash script (inspired from namcap)

* much better performance
A python module for calling vercmp natively, and the algorithm for checking
circular dependencies was greatly improved

* more accurate dependency and provision handling
Now versioned dependencies and provisions are handled correctly.

After building the python module and moving it next to the main script, it
should be possible to use it like this :

For core and extra :
./check_packages.py --abs-tree=/home/abs/rsync/i686 --repos=core,extra
./check_packages.py --abs-tree=/home/abs/rsync/x86_64 --repos=core,extra

For community :
./check_packages.py --abs-tree=/home/abs/rsync/i686 --repos=community
./check_packages.py --abs-tree=/home/abs/rsync/x86_64 --repos=community

Signed-off-by: Xavier Chantry <shiningxc@gmail.com>
Signed-off-by: Aaron Griffin <aaronmgriffin@gmail.com>
---
 cron-jobs/check_archlinux/README             |   8 +
 cron-jobs/check_archlinux/alpm.c             |  40 +++
 cron-jobs/check_archlinux/check_packages.py  | 392 +++++++++++++++++++++++++++
 cron-jobs/check_archlinux/parse_pkgbuilds.sh |  78 ++++++
 cron-jobs/check_archlinux/setup.py           |  10 +
 5 files changed, 528 insertions(+)
 create mode 100644 cron-jobs/check_archlinux/README
 create mode 100644 cron-jobs/check_archlinux/alpm.c
 create mode 100755 cron-jobs/check_archlinux/check_packages.py
 create mode 100755 cron-jobs/check_archlinux/parse_pkgbuilds.sh
 create mode 100644 cron-jobs/check_archlinux/setup.py

(limited to 'cron-jobs/check_archlinux')

diff --git a/cron-jobs/check_archlinux/README b/cron-jobs/check_archlinux/README
new file mode 100644
index 0000000..f3a1b90
--- /dev/null
+++ b/cron-jobs/check_archlinux/README
@@ -0,0 +1,8 @@
+1) Build the python module
+$ python setup.py build
+
+2) copy it back to the current working directory
+$ cp build/lib.*/alpm.* .
+
+3) run the script
+$ ./check_packages.py -h
diff --git a/cron-jobs/check_archlinux/alpm.c b/cron-jobs/check_archlinux/alpm.c
new file mode 100644
index 0000000..0b7cd2c
--- /dev/null
+++ b/cron-jobs/check_archlinux/alpm.c
@@ -0,0 +1,40 @@
+#include <Python.h>
+#include <alpm.h>
+
+static PyObject *
+alpm_vercmp(PyObject *self, PyObject *args)
+{
+	const char *v1, *v2;
+	int ret;
+
+	if (!PyArg_ParseTuple(args, "ss", &v1, &v2))
+		return NULL;
+	ret = alpm_pkg_vercmp(v1, v2);
+	return Py_BuildValue("i", ret);
+}
+
+static PyMethodDef AlpmMethods[] = {
+	{"vercmp",  alpm_vercmp, METH_VARARGS,
+		"Execute vercmp."},
+	{NULL, NULL, 0, NULL}        /* Sentinel */
+};
+
+PyMODINIT_FUNC
+initalpm(void)
+{
+	(void) Py_InitModule("alpm", AlpmMethods);
+}
+
+int
+main(int argc, char *argv[])
+{
+	/* Pass argv[0] to the Python interpreter */
+	Py_SetProgramName(argv[0]);
+
+	/* Initialize the Python interpreter.  Required. */
+	Py_Initialize();
+
+	/* Add a static module */
+	initalpm();
+	return 0;
+}
diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py
new file mode 100755
index 0000000..2d2efbe
--- /dev/null
+++ b/cron-jobs/check_archlinux/check_packages.py
@@ -0,0 +1,392 @@
+#!/usr/bin/python
+#
+# check_archlinux.py
+#
+# Original script by Scott Horowitz <stonecrest@gmail.com>
+# Rewritten by Xavier Chantry <shiningxc@gmail.com>
+#
+# This script currently checks for a number of issues in your ABS tree:
+#   1. Directories with missing PKGBUILDS
+#   2. Invalid PKGBUILDs (bash syntax error for instance)
+#   3. PKGBUILD names that don't match their directory
+#   4. Duplicate PKGBUILDs
+#   5. Valid arch's in PKGBUILDS
+#   6. Missing (make-)dependencies
+#   7. Hierarchy of repos (e.g., that a core package doesn't depend on
+#      a non-core package)
+#   8. Circular dependencies
+
+import os,re,commands,getopt,sys,alpm
+import pdb
+
+packages = {} # pkgname : PacmanPackage
+provisions = {} # provision : PacmanPackage
+pkgdeps,makepkgdeps = {},{} # pkgname : list of the PacmanPackage dependencies
+invalid_pkgbuilds = []
+missing_pkgbuilds = []
+dups = []
+
+mismatches = []
+missing_deps = []
+missing_makedeps = []
+invalid_archs = []
+dep_hierarchy = []
+makedep_hierarchy = []
+circular_deps = [] # pkgname>dep1>dep2>...>pkgname
+checked_deps = []
+
+class PacmanPackage:
+	def __init__(self):
+		self.name,self.version = "",""
+		self.path,self.repo = "",""
+		self.deps,self.makedeps = [],[]
+		self.provides,self.conflicts = [],[]
+		self.archs = []
+
+class Depend:
+	def __init__(self,name,version,mod):
+		self.name = name
+		self.version = version
+		self.mod = mod
+
+def parse_pkgbuilds(repos):
+	oldcwd = os.getcwd()
+	os.chdir(absroot)
+	for repo in repos:
+		data = commands.getoutput(oldcwd + '/parse_pkgbuilds.sh ' + repo)
+		parse_data(repo,data)
+	os.chdir(oldcwd)
+
+def parse_data(repo,data):
+	attrname = None
+
+	for line in data.split('\n'):
+		if line.startswith('%'):
+			attrname = line.strip('%').lower()
+		elif line.strip() == '':
+			attrname = None
+		elif attrname == "invalid":
+			if repo in repos:
+				invalid_pkgbuilds.append(line)
+		elif attrname == "missing":
+			if repo in repos:
+				missing_pkgbuilds.append(line)
+		elif attrname == "name":
+			pkg = PacmanPackage()
+			pkg.name = line
+			pkg.repo = repo
+			dup = None
+			if packages.has_key(pkg.name):
+				dup = packages[pkg.name]
+			packages[pkg.name] = pkg
+		elif attrname == "version":
+			pkg.version = line
+		elif attrname == "path":
+			pkg.path = line
+			if dup != None and (pkg.repo in repos or dup.repo in repos):
+				dups.append(pkg.path + " vs. " + dup.path)
+		elif attrname == "arch":
+			pkg.archs.append(line)
+		elif attrname == "depends":
+			pkg.deps.append(line)
+		elif attrname == "makedepends":
+			pkg.makedeps.append(line)
+		elif attrname == "conflicts":
+			pkg.conflicts.append(line)
+		elif attrname == "provides":
+			pkg.provides.append(line)
+			provname=line.split("=")[0]
+			if not provisions.has_key(provname):
+				provisions[provname] = []
+			provisions[provname].append(pkg)
+
+def splitdep(dep):
+	name = dep
+	version = ""
+	mod = ""
+	for char in (">=", "<=", "=", ">", "<"):
+		pos = dep.find(char)
+		if pos > -1:
+			name = dep[:pos]
+			version = dep[pos:].replace(char, "")
+			mod = char
+			break
+	return Depend(name,version,mod)
+
+def splitprov(prov):
+	name = prov
+	version = ""
+	pos = prov.find("=")
+	if pos > -1:
+		name = prov[:pos]
+		version = prov[pos:].replace("=", "")
+	return (name,version)
+
+def vercmp(v1,mod,v2):
+	res = alpm.vercmp(v1,v2)
+	if res == 0:
+		return (mod.find("=") > -1)
+	elif res < 0:
+		return (mod.find("<") > -1)
+	elif res > 0:
+		return (mod.find(">") > -1)
+	return False
+
+
+def depcmp(name,version,dep):
+	if name != dep.name:
+		return False
+	if dep.version == "" or dep.mod == "":
+		return True
+	if version == "":
+		return False
+	return vercmp(version,dep.mod,dep.version)
+
+def provcmp(pkg,dep):
+	for prov in pkg.provides:
+		(provname,provver) = splitprov(prov)
+		if depcmp(provname,provver,dep):
+			return True
+	return False
+
+def verify_dep(dep):
+	dep = splitdep(dep)
+	if packages.has_key(dep.name):
+		pkg = packages[dep.name]
+		if depcmp(pkg.name,pkg.version,dep):
+			return [pkg]
+	if provisions.has_key(dep.name):
+		provlist = provisions[dep.name]
+		results = []
+		for prov in provlist:
+			if provcmp(prov,dep):
+				results.append(prov)
+		return results
+	return []
+
+def verify_deps(name,repo,deps):
+	pkg_deps = []
+	missdeps = []
+	hierarchy = []
+	for dep in deps:
+		pkglist = verify_dep(dep)
+		if pkglist == []:
+			missdeps.append(name + " --> '" + dep + "'")
+		else:
+			valid_repos = get_repo_hierarchy(repo)
+			pkgdep = None
+			for pkg in pkglist:
+				if pkg.repo in valid_repos:
+					pkgdep = pkg
+					break
+			if not pkgdep:
+				pkgdep = pkglist[0]
+				hierarchy.append(repo + "/" + name + " depends on " + pkgdep.repo + "/" + pkgdep.name)
+			pkg_deps.append(pkgdep)
+
+	return (pkg_deps,missdeps,hierarchy)
+
+def get_repo_hierarchy(repo):
+	repo_hierarchy = {'core': ['core'], \
+		'extra': ['core', 'extra'], \
+		'community': ['core', 'extra', 'community']}
+	if repo_hierarchy.has_key(repo):
+		return repo_hierarchy[repo]
+	else:
+		return ['core','extra','community']
+
+def verify_archs(name,archs):
+	valid_archs = ['i686', 'x86_64']
+	invalid_archs = []
+	for arch in archs:
+		if arch not in valid_archs:
+			invalid_archs.append(name + " --> " + arch)
+	return invalid_archs
+
+def find_scc(packages):
+	# reset all variables
+	global index,S,pkgindex,pkglowlink
+	index = 0
+	S = []
+	pkgindex = {}
+	pkglowlink = {}
+	cycles = []
+	for pkg in packages:
+		tarjan(pkg)
+
+def tarjan(pkg):
+	global index,S,pkgindex,pkglowlink,cycles
+	pkgindex[pkg] = index
+	pkglowlink[pkg] = index
+	index += 1
+	checked_deps.append(pkg)
+	S.append(pkg)
+	if pkgdeps.has_key(pkg):
+		deps = pkgdeps[pkg]
+	else:
+		print pkg.name
+		deps = []
+	for dep in deps:
+		if not pkgindex.has_key(dep):
+			tarjan(dep)
+			pkglowlink[pkg] = min(pkglowlink[pkg],pkglowlink[dep])
+		elif dep in S:
+			pkglowlink[pkg] = min(pkglowlink[pkg],pkgindex[dep])
+	if pkglowlink[pkg] == pkgindex[pkg]:
+		dep = S.pop()
+		if pkg == dep:
+			return
+		path = pkg.name
+		while pkg != dep:
+			path = dep.name + ">" + path
+			dep = S.pop()
+		path = dep.name + ">" + path
+		if pkg.repo in repos:
+			circular_deps.append(path)
+
+def print_heading(heading):
+	print ""
+	print "=" * (len(heading) + 4)
+	print "= " + heading + " ="
+	print "=" * (len(heading) + 4)
+
+def print_subheading(subheading):
+	print ""
+	print subheading
+	print "-" * (len(subheading) + 2)
+
+def print_missdeps(pkgname,missdeps) :
+	for d in missdeps:
+		print pkgname + " : " + d
+
+def print_result(list, subheading):
+	if len(list) > 0:
+		print_subheading(subheading)
+		for item in list:
+			print item
+
+def print_results():
+	print_result(missing_pkgbuilds, "Missing PKGBUILDs")
+	print_result(invalid_pkgbuilds, "Invalid PKGBUILDs")
+	print_result(mismatches, "Mismatched Pkgnames")
+	print_result(dups, "Duplicate PKGBUILDs")
+	print_result(invalid_archs, "Invalid Archs")
+	print_result(missing_deps, "Missing Dependencies")
+	print_result(missing_makedeps, "Missing Makedepends")
+	print_result(dep_hierarchy, "Repo Hierarchy for Dependencies")
+	print_result(makedep_hierarchy, "Repo Hierarchy for Makedepends")
+	print_result(circular_deps, "Circular Dependencies")
+	print_subheading("Summary")
+	print "Missing PKGBUILDs:                    ", len(missing_pkgbuilds)
+	print "Invalid PKGBUILDs:                    ", len(invalid_pkgbuilds)
+	print "Mismatching PKGBUILD names:           ", len(mismatches)
+	print "Duplicate PKGBUILDs:                  ", len(dups)
+	print "Invalid archs:                        ", len(invalid_archs)
+	print "Missing (make)dependencies:           ", len(missing_deps)+len(missing_makedeps)
+	print "Repo hierarchy problems:              ", len(dep_hierarchy)+len(makedep_hierarchy)
+	print "Circular dependencies:                ", len(circular_deps)
+	print ""
+
+def print_usage():
+	print ""
+	print "Usage: ./check_packages.py [OPTION]"
+	print ""
+	print "Options:"
+	print "  --abs-tree=<path>             Check specified tree (default : /var/abs)"
+	print "  --repos=<r1,r2,...>           Check specified repos (default : core,extra)"
+	print "  -h, --help                    Show this help and exit"
+	print ""
+	print "Examples:"
+	print "\n  Check core and extra in existing abs tree:"
+	print "    ./check_packages.py --abs-tree=/var/abs --repos=core,extra"
+	print "\n  Check community:"
+	print "    ./check_packages.py --abs-tree=/var/abs --repos=community"
+	print ""
+
+## Default path to the abs root directory
+absroot = "/var/abs"
+## Default list of repos to check
+repos = ['core', 'extra']
+
+try:
+	opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos="])
+except getopt.GetoptError:
+	print_usage()
+	sys.exit()
+if opts != []:
+	for o, a in opts:
+		if o in ("--abs-tree"):
+			absroot = a
+		elif o in ("--repos"):
+			repos = a.split(",")
+		else:
+			print_usage()
+			sys.exit()
+		if args != []:
+			print_usage()
+			sys.exit()
+
+if not os.path.isdir(absroot):
+	print "Error : the abs tree " + absroot + " does not exist"
+	sys.exit()
+for repo in repos:
+	repopath = absroot + "/" + repo
+	if not os.path.isdir(repopath):
+		print "Error : the repository " + repo + " does not exist in " + absroot
+		sys.exit()
+# repos which need to be loaded
+loadrepos = set([])
+for repo in repos:
+	loadrepos = loadrepos | set(get_repo_hierarchy(repo))
+
+print_heading("Integrity Check")
+print "\nPerforming integrity checks..."
+
+print "==> parsing pkgbuilds"
+parse_pkgbuilds(loadrepos)
+
+repopkgs = {}
+for name,pkg in packages.iteritems():
+	if pkg.repo in repos:
+		repopkgs[name] = pkg
+
+print "==> checking mismatches"
+for name,pkg in repopkgs.iteritems():
+	pkgdirname = pkg.path.split("/")[-1]
+	if name != pkgdirname:
+		mismatches.append(name + " vs. " + pkg.path)
+
+print "==> checking archs"
+for name,pkg in repopkgs.iteritems():
+	archs = verify_archs(name,pkg.archs)
+	invalid_archs.extend(archs)
+
+# ugly hack to strip the weird kblic- deps
+for name,pkg in packages.iteritems():
+	p = re.compile('klibc-[A-Za-z0-9]{20,}|klibc-\*')
+	pkg.deps = [dep for dep in pkg.deps if not p.match(dep)]
+	pkg.makedeps = [dep for dep in pkg.makedeps if not p.match(dep)]
+
+print "==> checking dependencies"
+for name,pkg in repopkgs.iteritems():
+	(deps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.deps)
+	pkgdeps[pkg] = deps
+	missing_deps.extend(missdeps)
+	dep_hierarchy.extend(hierarchy)
+
+print "==> checking makedepends"
+for name,pkg in repopkgs.iteritems():
+	(makedeps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.makedeps)
+	makepkgdeps[pkg] = makedeps
+	missing_makedeps.extend(missdeps)
+	makedep_hierarchy.extend(hierarchy)
+
+print "==> checking for circular dependencies"
+# make sure pkgdeps is filled for every package
+for name,pkg in packages.iteritems():
+	if not pkgdeps.has_key(pkg):
+		(deps,missdeps,_) = verify_deps(name,pkg.repo,pkg.deps)
+		pkgdeps[pkg] = deps
+find_scc(repopkgs.values())
+
+print_results()
diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh
new file mode 100755
index 0000000..d4205ae
--- /dev/null
+++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh
@@ -0,0 +1,78 @@
+#!/bin/bash
+
+parse() {
+	unset pkgname pkgver pkgrel
+	unset depends makedepends conflicts provides
+	ret=0
+	dir=$1
+	pkgbuild=$dir/PKGBUILD
+	source $pkgbuild &>/dev/null || ret=$?
+
+	# ensure $pkgname and $pkgver variables were found
+	if [ $ret -ne 0 -o -z "$pkgname" -o -z "$pkgver" ]; then
+		echo -e "%INVALID%\n$pkgbuild\n"
+		return 1
+	fi 
+
+	echo -e "%NAME%\n$pkgname\n"
+	echo -e "%VERSION%\n$pkgver-$pkgrel\n"
+	echo -e "%PATH%\n$dir\n"
+
+	if [ -n "$arch" ]; then
+		echo "%ARCH%"
+		for i in ${arch[@]}; do echo $i; done
+		echo ""
+	fi
+	if [ -n "$depends" ]; then
+		echo "%DEPENDS%"
+		for i in ${depends[@]}; do
+			echo $i
+		done
+		echo ""
+	fi
+	if [ -n "$makedepends" ]; then
+		echo "%MAKEDEPENDS%"
+		for i in ${makedepends[@]}; do
+			echo $i
+		done
+		echo ""
+	fi
+	if [ -n "$conflicts" ]; then
+		echo "%CONFLICTS%"
+		for i in ${conflicts[@]}; do echo $i; done
+		echo ""
+	fi
+	if [ -n "$provides" ]; then
+		echo "%PROVIDES%"
+		for i in ${provides[@]}; do echo $i; done
+		echo ""
+	fi
+	return 0
+}
+
+find_pkgbuilds() {
+	if [ -f $1/PKGBUILD ]; then
+		parse $1
+		return
+	fi
+	empty=1
+	for dir in $1/*; do
+		if [ -d $dir ]; then
+			find_pkgbuilds $dir
+			unset empty
+		fi
+	done
+	if [ -n "$empty" ]; then
+		echo -e "%MISSING%\n$1\n"
+	fi
+}
+
+if [ -z "$*" ]; then
+	exit 1
+fi
+
+for dir in "$@"; do
+	find_pkgbuilds $dir
+done
+
+exit 0
diff --git a/cron-jobs/check_archlinux/setup.py b/cron-jobs/check_archlinux/setup.py
new file mode 100644
index 0000000..b172752
--- /dev/null
+++ b/cron-jobs/check_archlinux/setup.py
@@ -0,0 +1,10 @@
+from distutils.core import setup, Extension
+
+alpm = Extension('alpm',
+		libraries = ['alpm'],
+		sources = ['alpm.c'])
+
+setup (name = 'Alpm',
+		version = '1.0',
+		description = 'Alpm bindings',
+		ext_modules = [alpm])
-- 
cgit v1.2.3-2-g168b


From 86ec8b5c9dcd5b83cee55e410d42784c830efbb6 Mon Sep 17 00:00:00 2001
From: Xavier Chantry <shiningxc@gmail.com>
Date: Sat, 27 Sep 2008 00:56:24 +0200
Subject: check_packages : improve the "skip klibc dep" hack.

It is not possible to check the klibc dep on the PKGBUILD level, so I made a
hack to skip it.
This hack broke on klibc-jfflyAahxqaliwAofrf_fdf5upI because of the
underscore.
But we can simply use the \w regexp which matches any alpha-numeric char,
including underscore. According to klibc developer, dashes are possible too.
Also the length of this string is always 27 chars, so we can use that.

Signed-off-by: Xavier Chantry <shiningxc@gmail.com>
Signed-off-by: Aaron Griffin <aaronmgriffin@gmail.com>
---
 cron-jobs/check_archlinux/check_packages.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'cron-jobs/check_archlinux')

diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py
index 2d2efbe..69b14c3 100755
--- a/cron-jobs/check_archlinux/check_packages.py
+++ b/cron-jobs/check_archlinux/check_packages.py
@@ -363,7 +363,7 @@ for name,pkg in repopkgs.iteritems():
 
 # ugly hack to strip the weird kblic- deps
 for name,pkg in packages.iteritems():
-	p = re.compile('klibc-[A-Za-z0-9]{20,}|klibc-\*')
+	p = re.compile('klibc-[\w\-]{27}|klibc-\*')
 	pkg.deps = [dep for dep in pkg.deps if not p.match(dep)]
 	pkg.makedeps = [dep for dep in pkg.makedeps if not p.match(dep)]
 
-- 
cgit v1.2.3-2-g168b


From d39b68ede8567fb2ee15f45995ddab8f4710e9dc Mon Sep 17 00:00:00 2001
From: Xavier Chantry <shiningxc@gmail.com>
Date: Sat, 3 Jan 2009 13:22:57 +0100
Subject: check_packages : add --arch option.

The parsing script didn't set CARCH previously, and the flashplugin PKGBUILD
exited in this case.

First override the exit function to prevent the whole script to exit, and
add a --arch option to be able to set CARCH correctly.

To be used like this :

For core and extra :
./check_packages.py --abs-tree=/home/abs/rsync/i686 --repos=core,extra --arch=i686
./check_packages.py --abs-tree=/home/abs/rsync/x86_64 --repos=core,extra --arch=x86_64

For community :
./check_packages.py --abs-tree=/home/abs/rsync/i686 --repos=community --arch=i686
./check_packages.py --abs-tree=/home/abs/rsync/x86_64 --repos=community --arch=x86_64

Signed-off-by: Xavier Chantry <shiningxc@gmail.com>
Signed-off-by: Aaron Griffin <aaronmgriffin@gmail.com>
---
 cron-jobs/check_archlinux/check_packages.py  | 23 ++++++++++++++---------
 cron-jobs/check_archlinux/parse_pkgbuilds.sh |  9 ++++++++-
 2 files changed, 22 insertions(+), 10 deletions(-)

(limited to 'cron-jobs/check_archlinux')

diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py
index 69b14c3..f0fd4a5 100755
--- a/cron-jobs/check_archlinux/check_packages.py
+++ b/cron-jobs/check_archlinux/check_packages.py
@@ -49,11 +49,11 @@ class Depend:
 		self.version = version
 		self.mod = mod
 
-def parse_pkgbuilds(repos):
+def parse_pkgbuilds(repos,arch):
 	oldcwd = os.getcwd()
 	os.chdir(absroot)
 	for repo in repos:
-		data = commands.getoutput(oldcwd + '/parse_pkgbuilds.sh ' + repo)
+		data = commands.getoutput(oldcwd + '/parse_pkgbuilds.sh ' + arch + ' ' + repo)
 		parse_data(repo,data)
 	os.chdir(oldcwd)
 
@@ -292,24 +292,27 @@ def print_usage():
 	print "Usage: ./check_packages.py [OPTION]"
 	print ""
 	print "Options:"
-	print "  --abs-tree=<path>             Check specified tree (default : /var/abs)"
-	print "  --repos=<r1,r2,...>           Check specified repos (default : core,extra)"
+	print "  --abs-tree=<path>             Check the specified tree (default : /var/abs)"
+	print "  --repos=<r1,r2,...>           Check the specified repos (default : core,extra)"
+	print "  --arch=<i686|x86_64>          Check the specified arch (default : i686)"
 	print "  -h, --help                    Show this help and exit"
 	print ""
 	print "Examples:"
 	print "\n  Check core and extra in existing abs tree:"
-	print "    ./check_packages.py --abs-tree=/var/abs --repos=core,extra"
+	print "    ./check_packages.py --abs-tree=/var/abs --repos=core,extra --arch=i686"
 	print "\n  Check community:"
-	print "    ./check_packages.py --abs-tree=/var/abs --repos=community"
+	print "    ./check_packages.py --abs-tree=/var/abs --repos=community --arch=i686"
 	print ""
 
 ## Default path to the abs root directory
 absroot = "/var/abs"
 ## Default list of repos to check
 repos = ['core', 'extra']
+## Default arch
+arch = "i686"
 
 try:
-	opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos="])
+	opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos=", "arch="])
 except getopt.GetoptError:
 	print_usage()
 	sys.exit()
@@ -319,6 +322,8 @@ if opts != []:
 			absroot = a
 		elif o in ("--repos"):
 			repos = a.split(",")
+		elif o in ("--arch"):
+			arch = a
 		else:
 			print_usage()
 			sys.exit()
@@ -339,11 +344,11 @@ loadrepos = set([])
 for repo in repos:
 	loadrepos = loadrepos | set(get_repo_hierarchy(repo))
 
-print_heading("Integrity Check")
+print_heading("Integrity Check " + arch + " of " + ",".join(repos))
 print "\nPerforming integrity checks..."
 
 print "==> parsing pkgbuilds"
-parse_pkgbuilds(loadrepos)
+parse_pkgbuilds(loadrepos,arch)
 
 repopkgs = {}
 for name,pkg in packages.iteritems():
diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh
index d4205ae..47aec89 100755
--- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh
+++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh
@@ -1,5 +1,10 @@
 #!/bin/bash
 
+# Usage : parse_pkgbuilds.sh arch <pkgbuilds_dir1,dir2,...>
+# Example : parse_pkgbuilds.sh i686 /var/abs/core /var/abs/extra
+
+exit() { return; }
+
 parse() {
 	unset pkgname pkgver pkgrel
 	unset depends makedepends conflicts provides
@@ -67,10 +72,12 @@ find_pkgbuilds() {
 	fi
 }
 
-if [ -z "$*" ]; then
+if [ -z "$1" -o -z "$2" ]; then
 	exit 1
 fi
 
+CARCH=$1
+shift
 for dir in "$@"; do
 	find_pkgbuilds $dir
 done
-- 
cgit v1.2.3-2-g168b


From d9df504ddd648171eeff73d63afe196e153459a2 Mon Sep 17 00:00:00 2001
From: Xavier Chantry <shiningxc@gmail.com>
Date: Sun, 1 Feb 2009 20:59:10 +0100
Subject: check_packages: fix stupid assumption on script location.

The parse_pkgbuilds.sh script was assumed to be in the current working
directory, which is quite stupid since check_packages.py can be called from
anywhere. Now it only assumes that check_packages.py and parse_pkgbuilds.sh
are in the same directory.

This should fix the empty integrity checks on arch-dev :)

Signed-off-by: Xavier Chantry <shiningxc@gmail.com>
Signed-off-by: Dan McGee <dan@archlinux.org>
---
 cron-jobs/check_archlinux/check_packages.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

(limited to 'cron-jobs/check_archlinux')

diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py
index f0fd4a5..0392368 100755
--- a/cron-jobs/check_archlinux/check_packages.py
+++ b/cron-jobs/check_archlinux/check_packages.py
@@ -50,12 +50,10 @@ class Depend:
 		self.mod = mod
 
 def parse_pkgbuilds(repos,arch):
-	oldcwd = os.getcwd()
-	os.chdir(absroot)
 	for repo in repos:
-		data = commands.getoutput(oldcwd + '/parse_pkgbuilds.sh ' + arch + ' ' + repo)
+		data = commands.getoutput(os.path.dirname(sys.argv[0]) + '/parse_pkgbuilds.sh '
+				+ arch + ' ' + absroot + '/' +  repo)
 		parse_data(repo,data)
-	os.chdir(oldcwd)
 
 def parse_data(repo,data):
 	attrname = None
-- 
cgit v1.2.3-2-g168b


From 8e9a5b387a76f31796837e1000188d1152bd4b00 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fran=C3=A7ois=20Charette?= <francois@archlinux.org>
Date: Wed, 6 May 2009 12:32:43 -0700
Subject: check_packages.py: Add 'any' arch support

Signed-off-by: Aaron Griffin <aaronmgriffin@gmail.com>
---
 cron-jobs/check_archlinux/check_packages.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'cron-jobs/check_archlinux')

diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py
index 0392368..9bdd9c1 100755
--- a/cron-jobs/check_archlinux/check_packages.py
+++ b/cron-jobs/check_archlinux/check_packages.py
@@ -194,7 +194,7 @@ def get_repo_hierarchy(repo):
 		return ['core','extra','community']
 
 def verify_archs(name,archs):
-	valid_archs = ['i686', 'x86_64']
+	valid_archs = ['any', 'i686', 'x86_64']
 	invalid_archs = []
 	for arch in archs:
 		if arch not in valid_archs:
@@ -292,7 +292,7 @@ def print_usage():
 	print "Options:"
 	print "  --abs-tree=<path>             Check the specified tree (default : /var/abs)"
 	print "  --repos=<r1,r2,...>           Check the specified repos (default : core,extra)"
-	print "  --arch=<i686|x86_64>          Check the specified arch (default : i686)"
+	print "  --arch=<any|i686|x86_64>      Check the specified arch (default : i686)"
 	print "  -h, --help                    Show this help and exit"
 	print ""
 	print "Examples:"
-- 
cgit v1.2.3-2-g168b


From f515029a694a11065f95a79bd83b325bf04562bb Mon Sep 17 00:00:00 2001
From: Aaron Griffin <aaronmgriffin@gmail.com>
Date: Fri, 8 May 2009 13:21:39 -0700
Subject: check_archlinux: Skip searching CVS and .svn dirs

Prevents some errors in the community scripts due to
scanning of CVS dirs. Also skipping .svn dirs for the
future (dotglob may be set, in which case we'd scan
.svn dirs as well)

Signed-off-by: Aaron Griffin <aaronmgriffin@gmail.com>
---
 cron-jobs/check_archlinux/parse_pkgbuilds.sh | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'cron-jobs/check_archlinux')

diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh
index 47aec89..7dcfbbb 100755
--- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh
+++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh
@@ -56,6 +56,12 @@ parse() {
 }
 
 find_pkgbuilds() {
+    #Skip over some dirs
+    local d="$(basename $1)"
+    if [ "$d" = "CVS" -o "$d" = ".svn" ]; then
+        return
+    fi
+
 	if [ -f $1/PKGBUILD ]; then
 		parse $1
 		return
-- 
cgit v1.2.3-2-g168b


From d455a0b2cd72cd08b7b6cde878a29841f77a041e Mon Sep 17 00:00:00 2001
From: Xavier Chantry <shiningxc@gmail.com>
Date: Fri, 28 Aug 2009 01:27:11 +0200
Subject: check_packages : add support for split packages.

I just found a way to support split packages, by using $(type
package_${pkg}), parsing that output and running eval on the relevant lines.

This is a bit ugly, and while it works fine on my machine and my current abs
tree, I cannot guarantee this code is bug free :)

Signed-off-by: Xavier Chantry <shiningxc@gmail.com>
Signed-off-by: Aaron Griffin <aaronmgriffin@gmail.com>
---
 cron-jobs/check_archlinux/check_packages.py  |  5 +-
 cron-jobs/check_archlinux/parse_pkgbuilds.sh | 82 +++++++++++++++++++++++-----
 2 files changed, 73 insertions(+), 14 deletions(-)

(limited to 'cron-jobs/check_archlinux')

diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py
index 9bdd9c1..d42feab 100755
--- a/cron-jobs/check_archlinux/check_packages.py
+++ b/cron-jobs/check_archlinux/check_packages.py
@@ -38,6 +38,7 @@ checked_deps = []
 class PacmanPackage:
 	def __init__(self):
 		self.name,self.version = "",""
+		self.base = ""
 		self.path,self.repo = "",""
 		self.deps,self.makedeps = [],[]
 		self.provides,self.conflicts = [],[]
@@ -77,6 +78,8 @@ def parse_data(repo,data):
 			if packages.has_key(pkg.name):
 				dup = packages[pkg.name]
 			packages[pkg.name] = pkg
+		elif attrname == "base":
+			pkg.base = line
 		elif attrname == "version":
 			pkg.version = line
 		elif attrname == "path":
@@ -356,7 +359,7 @@ for name,pkg in packages.iteritems():
 print "==> checking mismatches"
 for name,pkg in repopkgs.iteritems():
 	pkgdirname = pkg.path.split("/")[-1]
-	if name != pkgdirname:
+	if name != pkgdirname and pkg.base != pkgdirname:
 		mismatches.append(name + " vs. " + pkg.path)
 
 print "==> checking archs"
diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh
index 7dcfbbb..0faa29f 100755
--- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh
+++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh
@@ -5,24 +5,36 @@
 
 exit() { return; }
 
-parse() {
-	unset pkgname pkgver pkgrel
-	unset depends makedepends conflicts provides
-	ret=0
-	dir=$1
-	pkgbuild=$dir/PKGBUILD
-	source $pkgbuild &>/dev/null || ret=$?
+variables=('pkgname' 'pkgver' 'pkgrel' 'depends' 'makedepends' 'provides' 'conflicts' )
+readonly -a variables
 
-	# ensure $pkgname and $pkgver variables were found
-	if [ $ret -ne 0 -o -z "$pkgname" -o -z "$pkgver" ]; then
-		echo -e "%INVALID%\n$pkgbuild\n"
-		return 1
-	fi 
+backup_package_variables() {
+	for var in ${variables[@]}; do
+		indirect="${var}_backup"
+		eval "${indirect}=(\${$var[@]})"
+	done
+}
+
+restore_package_variables() {
+	for var in ${variables[@]}; do
+		indirect="${var}_backup"
+		if [ -n "${!indirect}" ]; then
+			eval "${var}=(\${$indirect[@]})"
+		else
+			unset ${var}
+		fi
+	done
+}
 
+print_info() {
 	echo -e "%NAME%\n$pkgname\n"
 	echo -e "%VERSION%\n$pkgver-$pkgrel\n"
 	echo -e "%PATH%\n$dir\n"
 
+	if [ -n "$pkgbase" ]; then
+		echo -e "%BASE%\n$pkgbase\n"
+	fi
+
 	if [ -n "$arch" ]; then
 		echo "%ARCH%"
 		for i in ${arch[@]}; do echo $i; done
@@ -52,6 +64,50 @@ parse() {
 		for i in ${provides[@]}; do echo $i; done
 		echo ""
 	fi
+}
+
+source_pkgbuild() {
+	ret=0
+	dir=$1
+	pkgbuild=$dir/PKGBUILD
+	for var in ${variables[@]}; do
+		unset ${var}
+	done
+	source $pkgbuild &>/dev/null || ret=$?
+
+	# ensure $pkgname and $pkgver variables were found
+	if [ $ret -ne 0 -o -z "$pkgname" -o -z "$pkgver" ]; then
+		echo -e "%INVALID%\n$pkgbuild\n"
+		return 1
+	fi
+
+	if [ "${#pkgname[@]}" -gt "1" ]; then
+		for pkg in ${pkgname[@]}; do
+			if [ "$(type -t package_${pkg})" != "function" ]; then
+				echo -e "%INVALID%\n$pkgbuild\n"
+				return 1
+			else
+				backup_package_variables
+				pkgname=$pkg
+				while IFS= read -r line; do
+					var=${line%%=*}
+					var="${var#"${var%%[![:space:]]*}"}"   # remove leading whitespace characters
+					for realvar in ${variables[@]}; do
+						if [ "$var" == "$realvar" ]; then
+							eval $line
+							break
+						fi
+					done
+				done < <(type package_${pkg})
+				print_info
+				restore_package_variables
+			fi
+		done
+	else
+		echo
+		print_info
+	fi
+
 	return 0
 }
 
@@ -63,7 +119,7 @@ find_pkgbuilds() {
     fi
 
 	if [ -f $1/PKGBUILD ]; then
-		parse $1
+		source_pkgbuild $1
 		return
 	fi
 	empty=1
-- 
cgit v1.2.3-2-g168b


From 620b631fedbbd3596baa6752b9234249a2239a2a Mon Sep 17 00:00:00 2001
From: Xavier Chantry <shiningxc@gmail.com>
Date: Fri, 28 Aug 2009 01:27:12 +0200
Subject: add deplist to hierarchy problems

This should help a lot figuring out whether a given hierarchy problem is
fixable (if it doesn't require moving too many deps).

example output
core/crda depends on extra/python-m2crypto (27 extra (make)deps to pull)
core/iputils depends on extra/opensp (29 extra (make)deps to pull)
core/iputils depends on extra/libxslt (25 extra (make)deps to pull)
core/iputils depends on extra/docbook-xsl (27 extra (make)deps to pull)
core/udev depends on extra/gperf (0 extra (make)deps to pull)
core/udev depends on extra/libxslt (25 extra (make)deps to pull)
core/e2fsprogs depends on extra/bc (0 extra (make)deps to pull)
core/sqlite3 depends on extra/tcl (0 extra (make)deps to pull)
core/ca-certificates depends on extra/ruby (25 extra (make)deps to pull)

the actual deps are only displayed when there are less than 10.

Signed-off-by: Xavier Chantry <shiningxc@gmail.com>
Signed-off-by: Aaron Griffin <aaronmgriffin@gmail.com>
---
 cron-jobs/check_archlinux/check_packages.py | 50 +++++++++++++++++++++++++++--
 1 file changed, 47 insertions(+), 3 deletions(-)

(limited to 'cron-jobs/check_archlinux')

diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py
index d42feab..f2a9601 100755
--- a/cron-jobs/check_archlinux/check_packages.py
+++ b/cron-jobs/check_archlinux/check_packages.py
@@ -182,11 +182,50 @@ def verify_deps(name,repo,deps):
 					break
 			if not pkgdep:
 				pkgdep = pkglist[0]
-				hierarchy.append(repo + "/" + name + " depends on " + pkgdep.repo + "/" + pkgdep.name)
+				hierarchy.append((repo,name,pkgdep))
+
 			pkg_deps.append(pkgdep)
 
 	return (pkg_deps,missdeps,hierarchy)
 
+def compute_deplist_aux(pkg,deplist):
+	newdeplist = []
+	list = []
+	if pkgdeps.has_key(pkg):
+		list.extend(pkgdeps[pkg])
+	if makepkgdeps.has_key(pkg):
+		list.extend(makepkgdeps[pkg])
+	for dep in list:
+		if dep not in deplist:
+			newdeplist.append(dep)
+			deplist.append(dep)
+	for dep in newdeplist:
+		deplist2 = compute_deplist_aux(dep,deplist)
+		for dep2 in deplist2:
+			if dep2 not in deplist:
+				deplist.append(dep2)
+	return deplist
+
+def compute_deplist(pkg):
+	return compute_deplist_aux(pkg,[])
+
+def check_hierarchy(deph):
+	hierarchy = []
+	for (repo,name,pkgdep) in deph:
+		deplist = compute_deplist(pkgdep)
+		valid_repos = get_repo_hierarchy(repo)
+		extdeps = []
+		for dep in deplist:
+			if dep.repo not in valid_repos:
+				extdeps.append(dep.name)
+		string = repo + "/" + name + " depends on " + pkgdep.repo + "/" + pkgdep.name + " ("
+		string += "%s extra (make)deps to pull" % len(extdeps)
+		if 0 < len(extdeps) < 10:
+			string += " : " +  ' '.join(extdeps)
+		string += ")"
+		hierarchy.append(string)
+	return hierarchy
+
 def get_repo_hierarchy(repo):
 	repo_hierarchy = {'core': ['core'], \
 		'extra': ['core', 'extra'], \
@@ -373,19 +412,24 @@ for name,pkg in packages.iteritems():
 	pkg.deps = [dep for dep in pkg.deps if not p.match(dep)]
 	pkg.makedeps = [dep for dep in pkg.makedeps if not p.match(dep)]
 
+deph,makedeph = [],[]
+
 print "==> checking dependencies"
 for name,pkg in repopkgs.iteritems():
 	(deps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.deps)
 	pkgdeps[pkg] = deps
 	missing_deps.extend(missdeps)
-	dep_hierarchy.extend(hierarchy)
+	deph.extend(hierarchy)
 
 print "==> checking makedepends"
 for name,pkg in repopkgs.iteritems():
 	(makedeps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.makedeps)
 	makepkgdeps[pkg] = makedeps
 	missing_makedeps.extend(missdeps)
-	makedep_hierarchy.extend(hierarchy)
+	makedeph.extend(hierarchy)
+
+dep_hierarchy = check_hierarchy(deph)
+makedep_hierarchy = check_hierarchy(makedeph)
 
 print "==> checking for circular dependencies"
 # make sure pkgdeps is filled for every package
-- 
cgit v1.2.3-2-g168b


From a46bca9a39b37e7574f87468ddc5fb9b9577786e Mon Sep 17 00:00:00 2001
From: Henning Garus <henning.garus@gmail.com>
Date: Tue, 1 Sep 2009 23:54:47 +0200
Subject: check_packages.py: Allow multiple abs-trees

By parsing multiple abs trees we can add any when parsing the other trees,
checking any standalone doesn't make much sense.

Signed-off-by: Henning Garus <henning.garus@gmail.com>
Signed-off-by: Xavier Chantry <shiningxc@gmail.com>
Signed-off-by: Aaron Griffin <aaronmgriffin@gmail.com>
---
 cron-jobs/check_archlinux/check_packages.py | 32 +++++++++++++++--------------
 1 file changed, 17 insertions(+), 15 deletions(-)

(limited to 'cron-jobs/check_archlinux')

diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py
index f2a9601..c12858d 100755
--- a/cron-jobs/check_archlinux/check_packages.py
+++ b/cron-jobs/check_archlinux/check_packages.py
@@ -51,10 +51,11 @@ class Depend:
 		self.mod = mod
 
 def parse_pkgbuilds(repos,arch):
-	for repo in repos:
-		data = commands.getoutput(os.path.dirname(sys.argv[0]) + '/parse_pkgbuilds.sh '
-				+ arch + ' ' + absroot + '/' +  repo)
-		parse_data(repo,data)
+	for absroot in absroots:
+		for repo in repos:
+			data = commands.getoutput(os.path.dirname(sys.argv[0]) + '/parse_pkgbuilds.sh '
+					+ arch + ' ' + absroot + '/' +  repo)
+			parse_data(repo,data)
 
 def parse_data(repo,data):
 	attrname = None
@@ -332,9 +333,9 @@ def print_usage():
 	print "Usage: ./check_packages.py [OPTION]"
 	print ""
 	print "Options:"
-	print "  --abs-tree=<path>             Check the specified tree (default : /var/abs)"
+	print "  --abs-tree=<path[,path]>      Check the specified tree(s) (default : /var/abs)"
 	print "  --repos=<r1,r2,...>           Check the specified repos (default : core,extra)"
-	print "  --arch=<any|i686|x86_64>      Check the specified arch (default : i686)"
+	print "  --arch=<i686|x86_64>          Check the specified arch (default : i686)"
 	print "  -h, --help                    Show this help and exit"
 	print ""
 	print "Examples:"
@@ -345,7 +346,7 @@ def print_usage():
 	print ""
 
 ## Default path to the abs root directory
-absroot = "/var/abs"
+absroots = ["/var/abs"]
 ## Default list of repos to check
 repos = ['core', 'extra']
 ## Default arch
@@ -359,7 +360,7 @@ except getopt.GetoptError:
 if opts != []:
 	for o, a in opts:
 		if o in ("--abs-tree"):
-			absroot = a
+			absroot = a.split(',')
 		elif o in ("--repos"):
 			repos = a.split(",")
 		elif o in ("--arch"):
@@ -371,14 +372,15 @@ if opts != []:
 			print_usage()
 			sys.exit()
 
-if not os.path.isdir(absroot):
-	print "Error : the abs tree " + absroot + " does not exist"
-	sys.exit()
-for repo in repos:
-	repopath = absroot + "/" + repo
-	if not os.path.isdir(repopath):
-		print "Error : the repository " + repo + " does not exist in " + absroot
+for absroot in absroots:
+	if not os.path.isdir(absroot):
+		print "Error : the abs tree " + absroot + " does not exist"
 		sys.exit()
+	for repo in repos:
+		repopath = absroot + "/" + repo
+		if not os.path.isdir(repopath):
+			print "Error : the repository " + repo + " does not exist in " + absroot
+			sys.exit()
 # repos which need to be loaded
 loadrepos = set([])
 for repo in repos:
-- 
cgit v1.2.3-2-g168b


From ee0dd1ef6e9a0169ad54947758f2d7858a60d960 Mon Sep 17 00:00:00 2001
From: Henning Garus <henning.garus@gmail.com>
Date: Tue, 1 Sep 2009 23:57:39 +0200
Subject: Add db comparison to integrity check

Compare the abs tree with the repo dbs to check if we have a PKGBUILD for
each package in the dbs and vice versa.

Signed-off-by: Henning Garus <henning.garus@gmail.com>
Signed-off-by: Xavier Chantry <shiningxc@gmail.com>
Signed-off-by: Aaron Griffin <aaronmgriffin@gmail.com>
---
 cron-jobs/check_archlinux/check_packages.py | 55 +++++++++++++++++++++++++++--
 1 file changed, 53 insertions(+), 2 deletions(-)

(limited to 'cron-jobs/check_archlinux')

diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py
index c12858d..feab987 100755
--- a/cron-jobs/check_archlinux/check_packages.py
+++ b/cron-jobs/check_archlinux/check_packages.py
@@ -16,9 +16,11 @@
 #      a non-core package)
 #   8. Circular dependencies
 
-import os,re,commands,getopt,sys,alpm
+import os,re,commands,getopt,sys,tarfile,alpm
 import pdb
 
+DBEXT='.db.tar.gz'
+
 packages = {} # pkgname : PacmanPackage
 provisions = {} # provision : PacmanPackage
 pkgdeps,makepkgdeps = {},{} # pkgname : list of the PacmanPackage dependencies
@@ -26,6 +28,9 @@ invalid_pkgbuilds = []
 missing_pkgbuilds = []
 dups = []
 
+dbonly = []
+absonly = []
+
 mismatches = []
 missing_deps = []
 missing_makedeps = []
@@ -102,6 +107,17 @@ def parse_data(repo,data):
 				provisions[provname] = []
 			provisions[provname].append(pkg)
 
+def parse_dbs(repos,arch):
+	dbpkgs = {}
+	for repo in repos:
+		pkgs = set([])
+		db = tarfile.open(os.path.join(repodir,repo,'os',arch,repo + DBEXT))
+		for line in db.getnames():
+			if not '/' in line:
+				pkgs.add(line.rsplit('-',2)[0])
+		dbpkgs[repo] = pkgs
+	return(dbpkgs)
+
 def splitdep(dep):
 	name = dep
 	version = ""
@@ -317,6 +333,8 @@ def print_results():
 	print_result(dep_hierarchy, "Repo Hierarchy for Dependencies")
 	print_result(makedep_hierarchy, "Repo Hierarchy for Makedepends")
 	print_result(circular_deps, "Circular Dependencies")
+	print_result(dbonly, "Packages found in db, but not in tree")
+	print_result(absonly,"Packages found in tree, but not in db")
 	print_subheading("Summary")
 	print "Missing PKGBUILDs:                    ", len(missing_pkgbuilds)
 	print "Invalid PKGBUILDs:                    ", len(invalid_pkgbuilds)
@@ -326,6 +344,8 @@ def print_results():
 	print "Missing (make)dependencies:           ", len(missing_deps)+len(missing_makedeps)
 	print "Repo hierarchy problems:              ", len(dep_hierarchy)+len(makedep_hierarchy)
 	print "Circular dependencies:                ", len(circular_deps)
+	print "In db, but not in tree:               ", len(dbonly)
+	print "In tree, but not in db:               ", len(absonly)
 	print ""
 
 def print_usage():
@@ -336,6 +356,7 @@ def print_usage():
 	print "  --abs-tree=<path[,path]>      Check the specified tree(s) (default : /var/abs)"
 	print "  --repos=<r1,r2,...>           Check the specified repos (default : core,extra)"
 	print "  --arch=<i686|x86_64>          Check the specified arch (default : i686)"
+	print "  --repo-dir=<path>             Check the dbs at the specified path (default : /srv/ftp)"
 	print "  -h, --help                    Show this help and exit"
 	print ""
 	print "Examples:"
@@ -351,9 +372,12 @@ absroots = ["/var/abs"]
 repos = ['core', 'extra']
 ## Default arch
 arch = "i686"
+## Default repodir
+repodir = "/srv/ftp"
 
 try:
-	opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos=", "arch="])
+	opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos=",
+	"arch=", "repo-dir="])
 except getopt.GetoptError:
 	print_usage()
 	sys.exit()
@@ -365,6 +389,8 @@ if opts != []:
 			repos = a.split(",")
 		elif o in ("--arch"):
 			arch = a
+		elif o in ("--repo-dir"):
+			repodir = a
 		else:
 			print_usage()
 			sys.exit()
@@ -381,6 +407,17 @@ for absroot in absroots:
 		if not os.path.isdir(repopath):
 			print "Error : the repository " + repo + " does not exist in " + absroot
 			sys.exit()
+if not os.path.isdir(repodir):
+	print "Error: the repository directory %s does not exist" % repodir
+	sys.exit()
+for repo in repos:
+	path = os.path.join(repodir,repo,'os',arch,repo + DBEXT)
+	if not os.path.isfile(path):
+		print "Error : repo DB %s : File not found" % path
+		sys.exit()
+	if not tarfile.is_tarfile(path):
+		print "Error : Cant open repo DB %s, not a valid tar file" % path
+		sys.exit()
 # repos which need to be loaded
 loadrepos = set([])
 for repo in repos:
@@ -397,6 +434,9 @@ for name,pkg in packages.iteritems():
 	if pkg.repo in repos:
 		repopkgs[name] = pkg
 
+print "==> parsing db files"
+dbpkgs = parse_dbs(repos,arch)
+
 print "==> checking mismatches"
 for name,pkg in repopkgs.iteritems():
 	pkgdirname = pkg.path.split("/")[-1]
@@ -441,4 +481,15 @@ for name,pkg in packages.iteritems():
 		pkgdeps[pkg] = deps
 find_scc(repopkgs.values())
 
+print "==> checking for differences between db files and pkgbuilds"
+for repo in repos:
+	for pkg in dbpkgs[repo]:
+		if not (pkg in repopkgs.keys() and repopkgs[pkg].repo == repo):
+			dbonly.append("%s/%s" % (repo,pkg))
+dbonly.sort()
+for name,pkg in repopkgs.iteritems():
+	if not name in dbpkgs[pkg.repo]:
+		absonly.append("%s/%s" % (pkg.repo,name))
+absonly.sort()
+
 print_results()
-- 
cgit v1.2.3-2-g168b


From ea827b50d0c65b4ba9f2e2d3c0761572915e6b03 Mon Sep 17 00:00:00 2001
From: Xavier Chantry <shiningxc@gmail.com>
Date: Wed, 2 Sep 2009 18:29:19 +0200
Subject: add the repo/ info to most results, and sort them

Add repo information to most results by using the repo/pkgname syntax
(instead of only pkgname)

Sort all results list (with the point above, this sorts the results by
repo :))

Signed-off-by: Xavier Chantry <shiningxc@gmail.com>
Signed-off-by: Aaron Griffin <aaronmgriffin@gmail.com>
---
 cron-jobs/check_archlinux/check_packages.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

(limited to 'cron-jobs/check_archlinux')

diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py
index feab987..96f3181 100755
--- a/cron-jobs/check_archlinux/check_packages.py
+++ b/cron-jobs/check_archlinux/check_packages.py
@@ -189,7 +189,7 @@ def verify_deps(name,repo,deps):
 	for dep in deps:
 		pkglist = verify_dep(dep)
 		if pkglist == []:
-			missdeps.append(name + " --> '" + dep + "'")
+			missdeps.append(repo + "/" + name + " --> '" + dep + "'")
 		else:
 			valid_repos = get_repo_hierarchy(repo)
 			pkgdep = None
@@ -252,12 +252,12 @@ def get_repo_hierarchy(repo):
 	else:
 		return ['core','extra','community']
 
-def verify_archs(name,archs):
+def verify_archs(name,repo,archs):
 	valid_archs = ['any', 'i686', 'x86_64']
 	invalid_archs = []
 	for arch in archs:
 		if arch not in valid_archs:
-			invalid_archs.append(name + " --> " + arch)
+			invalid_archs.append(repo + "/" + name + " --> " + arch)
 	return invalid_archs
 
 def find_scc(packages):
@@ -295,7 +295,7 @@ def tarjan(pkg):
 			return
 		path = pkg.name
 		while pkg != dep:
-			path = dep.name + ">" + path
+			path = dep.repo + "/" + dep.name + ">" + path
 			dep = S.pop()
 		path = dep.name + ">" + path
 		if pkg.repo in repos:
@@ -318,6 +318,7 @@ def print_missdeps(pkgname,missdeps) :
 
 def print_result(list, subheading):
 	if len(list) > 0:
+		list.sort()
 		print_subheading(subheading)
 		for item in list:
 			print item
@@ -445,7 +446,7 @@ for name,pkg in repopkgs.iteritems():
 
 print "==> checking archs"
 for name,pkg in repopkgs.iteritems():
-	archs = verify_archs(name,pkg.archs)
+	archs = verify_archs(name,pkg.repo,pkg.archs)
 	invalid_archs.extend(archs)
 
 # ugly hack to strip the weird kblic- deps
@@ -486,10 +487,8 @@ for repo in repos:
 	for pkg in dbpkgs[repo]:
 		if not (pkg in repopkgs.keys() and repopkgs[pkg].repo == repo):
 			dbonly.append("%s/%s" % (repo,pkg))
-dbonly.sort()
 for name,pkg in repopkgs.iteritems():
 	if not name in dbpkgs[pkg.repo]:
 		absonly.append("%s/%s" % (pkg.repo,name))
-absonly.sort()
 
 print_results()
-- 
cgit v1.2.3-2-g168b


From fe4152a6949a9ec3132d1a1c08335f370989198f Mon Sep 17 00:00:00 2001
From: Xavier Chantry <shiningxc@gmail.com>
Date: Thu, 3 Sep 2009 22:34:39 +0200
Subject: improve compute_dep algorithm

The compute_dep function I wrote for the more informative hierarchy output
was very inefficient. It's much better now (10s -> 0.5s) on my box, and I
get exactly the same results :)

Now the big majority of the time is again spent on parsing pkgbuilds.

Signed-off-by: Xavier Chantry <shiningxc@gmail.com>
Signed-off-by: Aaron Griffin <aaronmgriffin@gmail.com>
---
 cron-jobs/check_archlinux/check_packages.py | 35 +++++++++++++----------------
 1 file changed, 16 insertions(+), 19 deletions(-)

(limited to 'cron-jobs/check_archlinux')

diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py
index 96f3181..9ffdac5 100755
--- a/cron-jobs/check_archlinux/check_packages.py
+++ b/cron-jobs/check_archlinux/check_packages.py
@@ -205,26 +205,22 @@ def verify_deps(name,repo,deps):
 
 	return (pkg_deps,missdeps,hierarchy)
 
-def compute_deplist_aux(pkg,deplist):
-	newdeplist = []
-	list = []
-	if pkgdeps.has_key(pkg):
-		list.extend(pkgdeps[pkg])
-	if makepkgdeps.has_key(pkg):
-		list.extend(makepkgdeps[pkg])
-	for dep in list:
-		if dep not in deplist:
-			newdeplist.append(dep)
-			deplist.append(dep)
-	for dep in newdeplist:
-		deplist2 = compute_deplist_aux(dep,deplist)
-		for dep2 in deplist2:
-			if dep2 not in deplist:
-				deplist.append(dep2)
-	return deplist
-
 def compute_deplist(pkg):
-	return compute_deplist_aux(pkg,[])
+	list = []
+	stack = [pkg]
+	while stack != []:
+		dep = stack.pop()
+		if pkgdeps.has_key(dep):
+			for dep2 in pkgdeps[dep]:
+				if dep2 not in list:
+					list.append(dep2)
+					stack.append(dep2)
+		if makepkgdeps.has_key(dep):
+			for dep2 in makepkgdeps[dep]:
+				if dep2 not in list:
+					list.append(dep2)
+					stack.append(dep2)
+	return list
 
 def check_hierarchy(deph):
 	hierarchy = []
@@ -471,6 +467,7 @@ for name,pkg in repopkgs.iteritems():
 	missing_makedeps.extend(missdeps)
 	makedeph.extend(hierarchy)
 
+print "==> checking hierarchy"
 dep_hierarchy = check_hierarchy(deph)
 makedep_hierarchy = check_hierarchy(makedeph)
 
-- 
cgit v1.2.3-2-g168b


From b36b754497df69778273f6a6e7b0d91679646565 Mon Sep 17 00:00:00 2001
From: Xavier Chantry <shiningxc@gmail.com>
Date: Thu, 3 Sep 2009 23:34:24 +0200
Subject: parse_pkgbuilds : simple performance tweak

this simple tweak gives a nice perf boost : from 10s to 7s to parse extra
repo.

indeed calling basename caused a fork to happen for the thousand of files
being considered

Now the major bottleneck is parsing split pkgbuilds, but I am afraid there
is no magical solution for that :(

Signed-off-by: Xavier Chantry <shiningxc@gmail.com>
Signed-off-by: Aaron Griffin <aaronmgriffin@gmail.com>
---
 cron-jobs/check_archlinux/parse_pkgbuilds.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'cron-jobs/check_archlinux')

diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh
index 0faa29f..c2df7d1 100755
--- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh
+++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh
@@ -113,7 +113,7 @@ source_pkgbuild() {
 
 find_pkgbuilds() {
     #Skip over some dirs
-    local d="$(basename $1)"
+    local d="${1##*/}"
     if [ "$d" = "CVS" -o "$d" = ".svn" ]; then
         return
     fi
-- 
cgit v1.2.3-2-g168b


From 7eccf95bb5d53cb1b4f8c1d363cd5645ca4e567e Mon Sep 17 00:00:00 2001
From: Xavier Chantry <shiningxc@gmail.com>
Date: Sun, 6 Sep 2009 21:36:42 +0200
Subject: improve the way parse_pkgbuilds.sh is called

1) check the return value. before a failed call to parse_pkgbuilds.sh was
completely silent.. not good :)

2) call realpath on check_packages.py before determining the directory name,
to figure out where parse_pkgbuilds.sh is. This allows to symlink
check_packages.py and use the symlink without moving parse_pkgbuilds.sh .

for example : /foo/bar/ contains check_packages.py and parse_pkgbuilds.sh
ln -s /foo/bar/check_packages.py /bin/check_package

dirname(/bin/check_package) returns /bin/ so not good
dirname(realpath(/bin/check_package)) returns /foo/bar/ so good

Signed-off-by: Xavier Chantry <shiningxc@gmail.com>
Signed-off-by: Aaron Griffin <aaronmgriffin@gmail.com>
---
 cron-jobs/check_archlinux/check_packages.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

(limited to 'cron-jobs/check_archlinux')

diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py
index 9ffdac5..cb7eae4 100755
--- a/cron-jobs/check_archlinux/check_packages.py
+++ b/cron-jobs/check_archlinux/check_packages.py
@@ -58,9 +58,13 @@ class Depend:
 def parse_pkgbuilds(repos,arch):
 	for absroot in absroots:
 		for repo in repos:
-			data = commands.getoutput(os.path.dirname(sys.argv[0]) + '/parse_pkgbuilds.sh '
-					+ arch + ' ' + absroot + '/' +  repo)
-			parse_data(repo,data)
+			cmd = os.path.dirname(os.path.realpath(sys.argv[0])) + '/parse_pkgbuilds.sh '
+			cmd += arch + ' ' + absroot + '/' + repo
+			(status,output) = commands.getstatusoutput(cmd)
+			if status != 0:
+				print "Error : failed to run '%s'" % cmd
+				sys.exit()
+			parse_data(repo,output)
 
 def parse_data(repo,data):
 	attrname = None
-- 
cgit v1.2.3-2-g168b


From 12aba89ed2100552ddfe7d69bbd9e35a2019d098 Mon Sep 17 00:00:00 2001
From: Xavier Chantry <shiningxc@gmail.com>
Date: Tue, 15 Sep 2009 09:19:02 +0200
Subject: parse_pkgbuilds.sh : fix pkgbase handling

add the default value for pkgbase

also rework slightly how variables are handled (better distinction between
splitpkg variables and the rest)

Signed-off-by: Xavier Chantry <shiningxc@gmail.com>
Signed-off-by: Aaron Griffin <aaronmgriffin@gmail.com>
---
 cron-jobs/check_archlinux/parse_pkgbuilds.sh | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

(limited to 'cron-jobs/check_archlinux')

diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh
index c2df7d1..5cd17e4 100755
--- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh
+++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh
@@ -5,18 +5,19 @@
 
 exit() { return; }
 
-variables=('pkgname' 'pkgver' 'pkgrel' 'depends' 'makedepends' 'provides' 'conflicts' )
-readonly -a variables
+splitpkg_overrides=('depends' 'optdepends' 'provides' 'conflicts')
+variables=('pkgname' 'pkgbase' 'pkgver' 'pkgrel' 'makedepends' 'arch' ${splitpkg_overrides[@]})
+readonly -a variables splitpkg_overrides
 
 backup_package_variables() {
-	for var in ${variables[@]}; do
+	for var in ${splitpkg_overrides[@]}; do
 		indirect="${var}_backup"
 		eval "${indirect}=(\${$var[@]})"
 	done
 }
 
 restore_package_variables() {
-	for var in ${variables[@]}; do
+	for var in ${splitpkg_overrides[@]}; do
 		indirect="${var}_backup"
 		if [ -n "${!indirect}" ]; then
 			eval "${var}=(\${$indirect[@]})"
@@ -82,6 +83,7 @@ source_pkgbuild() {
 	fi
 
 	if [ "${#pkgname[@]}" -gt "1" ]; then
+		pkgbase=${pkgbase:-${pkgname[0]}}
 		for pkg in ${pkgname[@]}; do
 			if [ "$(type -t package_${pkg})" != "function" ]; then
 				echo -e "%INVALID%\n$pkgbuild\n"
-- 
cgit v1.2.3-2-g168b


From 43f4ada2f207ebf6d3cdd95d511e3ecdc01e6d7a Mon Sep 17 00:00:00 2001
From: Aaron Griffin <aaronmgriffin@gmail.com>
Date: Fri, 18 Sep 2009 17:22:55 -0400
Subject: Bump check_archlinux module for libalpm.so.4

Signed-off-by: Aaron Griffin <aaronmgriffin@gmail.com>
---
 cron-jobs/check_archlinux/alpm.so | Bin 0 -> 7238 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100755 cron-jobs/check_archlinux/alpm.so

(limited to 'cron-jobs/check_archlinux')

diff --git a/cron-jobs/check_archlinux/alpm.so b/cron-jobs/check_archlinux/alpm.so
new file mode 100755
index 0000000..b87b894
Binary files /dev/null and b/cron-jobs/check_archlinux/alpm.so differ
-- 
cgit v1.2.3-2-g168b


From 85eee9a95d2fcbdce483b94202b489d07d258690 Mon Sep 17 00:00:00 2001
From: Aaron Griffin <aaronmgriffin@gmail.com>
Date: Fri, 18 Sep 2009 17:28:46 -0400
Subject: check_archlinux: absroot -> absroots

Allow us to use the --abs-root flag. He he

Signed-off-by: Aaron Griffin <aaronmgriffin@gmail.com>
---
 cron-jobs/check_archlinux/check_packages.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'cron-jobs/check_archlinux')

diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py
index cb7eae4..25362c0 100755
--- a/cron-jobs/check_archlinux/check_packages.py
+++ b/cron-jobs/check_archlinux/check_packages.py
@@ -385,7 +385,7 @@ except getopt.GetoptError:
 if opts != []:
 	for o, a in opts:
 		if o in ("--abs-tree"):
-			absroot = a.split(',')
+			absroots = a.split(',')
 		elif o in ("--repos"):
 			repos = a.split(",")
 		elif o in ("--arch"):
-- 
cgit v1.2.3-2-g168b


From 4643d931f6adc4f7f55c7c2fbb37532614742c74 Mon Sep 17 00:00:00 2001
From: Xavier Chantry <shiningxc@gmail.com>
Date: Sun, 20 Sep 2009 14:50:34 +0200
Subject: check_packages : fix a bug when filling provisions

When we had a duplicate package, we would replace the old ref by the new
ref. But the provisions dict kept both the old and new ref. This was not
good at all.

Now, we just keep the old ref, and we only fill provisions after all
packages have been parsed. This should be much more sane.

Signed-off-by: Xavier Chantry <shiningxc@gmail.com>
Signed-off-by: Aaron Griffin <aaronmgriffin@gmail.com>
---
 cron-jobs/check_archlinux/check_packages.py | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

(limited to 'cron-jobs/check_archlinux')

diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py
index 25362c0..e8830e0 100755
--- a/cron-jobs/check_archlinux/check_packages.py
+++ b/cron-jobs/check_archlinux/check_packages.py
@@ -22,8 +22,9 @@ import pdb
 DBEXT='.db.tar.gz'
 
 packages = {} # pkgname : PacmanPackage
+repopkgs = {} # pkgname : PacmanPackage
 provisions = {} # provision : PacmanPackage
-pkgdeps,makepkgdeps = {},{} # pkgname : list of the PacmanPackage dependencies
+pkgdeps,makepkgdeps = {},{} # PacmanPackage : list of the PacmanPackage dependencies
 invalid_pkgbuilds = []
 missing_pkgbuilds = []
 dups = []
@@ -87,7 +88,8 @@ def parse_data(repo,data):
 			dup = None
 			if packages.has_key(pkg.name):
 				dup = packages[pkg.name]
-			packages[pkg.name] = pkg
+			else:
+				packages[pkg.name] = pkg
 		elif attrname == "base":
 			pkg.base = line
 		elif attrname == "version":
@@ -106,10 +108,6 @@ def parse_data(repo,data):
 			pkg.conflicts.append(line)
 		elif attrname == "provides":
 			pkg.provides.append(line)
-			provname=line.split("=")[0]
-			if not provisions.has_key(provname):
-				provisions[provname] = []
-			provisions[provname].append(pkg)
 
 def parse_dbs(repos,arch):
 	dbpkgs = {}
@@ -278,11 +276,9 @@ def tarjan(pkg):
 	index += 1
 	checked_deps.append(pkg)
 	S.append(pkg)
+	deps = []
 	if pkgdeps.has_key(pkg):
 		deps = pkgdeps[pkg]
-	else:
-		print pkg.name
-		deps = []
 	for dep in deps:
 		if not pkgindex.has_key(dep):
 			tarjan(dep)
@@ -430,7 +426,15 @@ print "\nPerforming integrity checks..."
 print "==> parsing pkgbuilds"
 parse_pkgbuilds(loadrepos,arch)
 
-repopkgs = {}
+# fill provisions
+for name,pkg in packages.iteritems():
+	for prov in pkg.provides:
+		provname=prov.split("=")[0]
+		if not provisions.has_key(provname):
+			provisions[provname] = []
+		provisions[provname].append(pkg)
+
+# fill repopkgs
 for name,pkg in packages.iteritems():
 	if pkg.repo in repos:
 		repopkgs[name] = pkg
-- 
cgit v1.2.3-2-g168b


From 9c624b099eb16e27c80ed8ff5f551ce6e8ad0682 Mon Sep 17 00:00:00 2001
From: Pierre Schmitz <pierre@archlinux.de>
Date: Mon, 22 Nov 2010 22:21:37 +0100
Subject: check_archlinux: fix for python2 usage

---
 cron-jobs/check_archlinux/README            |   2 +-
 cron-jobs/check_archlinux/alpm.so           | Bin 7238 -> 7166 bytes
 cron-jobs/check_archlinux/check_packages.py |   2 +-
 3 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'cron-jobs/check_archlinux')

diff --git a/cron-jobs/check_archlinux/README b/cron-jobs/check_archlinux/README
index f3a1b90..8f178dc 100644
--- a/cron-jobs/check_archlinux/README
+++ b/cron-jobs/check_archlinux/README
@@ -1,5 +1,5 @@
 1) Build the python module
-$ python setup.py build
+$ python2 setup.py build
 
 2) copy it back to the current working directory
 $ cp build/lib.*/alpm.* .
diff --git a/cron-jobs/check_archlinux/alpm.so b/cron-jobs/check_archlinux/alpm.so
index b87b894..50f0f97 100755
Binary files a/cron-jobs/check_archlinux/alpm.so and b/cron-jobs/check_archlinux/alpm.so differ
diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py
index e8830e0..465d87f 100755
--- a/cron-jobs/check_archlinux/check_packages.py
+++ b/cron-jobs/check_archlinux/check_packages.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python2
 #
 # check_archlinux.py
 #
-- 
cgit v1.2.3-2-g168b


From 639bef5d42d64871bc332c5618e29d457962d5f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9my=20Oudompheng?= <oudomphe@math.unice.fr>
Date: Sat, 4 Dec 2010 18:09:18 +0100
Subject: Style changes: remove usage of the 'has_key' method.

Signed-off-by: Pierre Schmitz <pierre@archlinux.de>
---
 cron-jobs/check_archlinux/check_packages.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

(limited to 'cron-jobs/check_archlinux')

diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py
index 465d87f..3874937 100755
--- a/cron-jobs/check_archlinux/check_packages.py
+++ b/cron-jobs/check_archlinux/check_packages.py
@@ -86,7 +86,7 @@ def parse_data(repo,data):
 			pkg.name = line
 			pkg.repo = repo
 			dup = None
-			if packages.has_key(pkg.name):
+			if pkg.name in packages:
 				dup = packages[pkg.name]
 			else:
 				packages[pkg.name] = pkg
@@ -171,11 +171,11 @@ def provcmp(pkg,dep):
 
 def verify_dep(dep):
 	dep = splitdep(dep)
-	if packages.has_key(dep.name):
+	if dep.name in packages:
 		pkg = packages[dep.name]
 		if depcmp(pkg.name,pkg.version,dep):
 			return [pkg]
-	if provisions.has_key(dep.name):
+	if dep.name in provisions:
 		provlist = provisions[dep.name]
 		results = []
 		for prov in provlist:
@@ -212,12 +212,12 @@ def compute_deplist(pkg):
 	stack = [pkg]
 	while stack != []:
 		dep = stack.pop()
-		if pkgdeps.has_key(dep):
+		if dep in pkgdeps:
 			for dep2 in pkgdeps[dep]:
 				if dep2 not in list:
 					list.append(dep2)
 					stack.append(dep2)
-		if makepkgdeps.has_key(dep):
+		if dep in makepkgdeps:
 			for dep2 in makepkgdeps[dep]:
 				if dep2 not in list:
 					list.append(dep2)
@@ -245,7 +245,7 @@ def get_repo_hierarchy(repo):
 	repo_hierarchy = {'core': ['core'], \
 		'extra': ['core', 'extra'], \
 		'community': ['core', 'extra', 'community']}
-	if repo_hierarchy.has_key(repo):
+	if repo in repo_hierarchy:
 		return repo_hierarchy[repo]
 	else:
 		return ['core','extra','community']
@@ -277,10 +277,10 @@ def tarjan(pkg):
 	checked_deps.append(pkg)
 	S.append(pkg)
 	deps = []
-	if pkgdeps.has_key(pkg):
+	if pkg in pkgdeps:
 		deps = pkgdeps[pkg]
 	for dep in deps:
-		if not pkgindex.has_key(dep):
+		if dep not in pkgindex:
 			tarjan(dep)
 			pkglowlink[pkg] = min(pkglowlink[pkg],pkglowlink[dep])
 		elif dep in S:
@@ -430,7 +430,7 @@ parse_pkgbuilds(loadrepos,arch)
 for name,pkg in packages.iteritems():
 	for prov in pkg.provides:
 		provname=prov.split("=")[0]
-		if not provisions.has_key(provname):
+		if provname not in provisions:
 			provisions[provname] = []
 		provisions[provname].append(pkg)
 
@@ -482,7 +482,7 @@ makedep_hierarchy = check_hierarchy(makedeph)
 print "==> checking for circular dependencies"
 # make sure pkgdeps is filled for every package
 for name,pkg in packages.iteritems():
-	if not pkgdeps.has_key(pkg):
+	if pkg not in pkgdeps:
 		(deps,missdeps,_) = verify_deps(name,pkg.repo,pkg.deps)
 		pkgdeps[pkg] = deps
 find_scc(repopkgs.values())
@@ -490,7 +490,7 @@ find_scc(repopkgs.values())
 print "==> checking for differences between db files and pkgbuilds"
 for repo in repos:
 	for pkg in dbpkgs[repo]:
-		if not (pkg in repopkgs.keys() and repopkgs[pkg].repo == repo):
+		if not (pkg in repopkgs and repopkgs[pkg].repo == repo):
 			dbonly.append("%s/%s" % (repo,pkg))
 for name,pkg in repopkgs.iteritems():
 	if not name in dbpkgs[pkg.repo]:
-- 
cgit v1.2.3-2-g168b


From dd5b341abea441a6a3e729afc6d1231bc0b0abe3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9my=20Oudompheng?= <oudomphe@math.unice.fr>
Date: Sat, 4 Dec 2010 18:19:45 +0100
Subject: Style changes: add parentheses around print "...".

Signed-off-by: Pierre Schmitz <pierre@archlinux.de>
---
 cron-jobs/check_archlinux/check_packages.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

(limited to 'cron-jobs/check_archlinux')

diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py
index 3874937..948a61a 100755
--- a/cron-jobs/check_archlinux/check_packages.py
+++ b/cron-jobs/check_archlinux/check_packages.py
@@ -421,9 +421,9 @@ for repo in repos:
 	loadrepos = loadrepos | set(get_repo_hierarchy(repo))
 
 print_heading("Integrity Check " + arch + " of " + ",".join(repos))
-print "\nPerforming integrity checks..."
+print("\nPerforming integrity checks...")
 
-print "==> parsing pkgbuilds"
+print("==> parsing pkgbuilds")
 parse_pkgbuilds(loadrepos,arch)
 
 # fill provisions
@@ -439,16 +439,16 @@ for name,pkg in packages.iteritems():
 	if pkg.repo in repos:
 		repopkgs[name] = pkg
 
-print "==> parsing db files"
+print("==> parsing db files")
 dbpkgs = parse_dbs(repos,arch)
 
-print "==> checking mismatches"
+print("==> checking mismatches")
 for name,pkg in repopkgs.iteritems():
 	pkgdirname = pkg.path.split("/")[-1]
 	if name != pkgdirname and pkg.base != pkgdirname:
 		mismatches.append(name + " vs. " + pkg.path)
 
-print "==> checking archs"
+print("==> checking archs")
 for name,pkg in repopkgs.iteritems():
 	archs = verify_archs(name,pkg.repo,pkg.archs)
 	invalid_archs.extend(archs)
@@ -461,25 +461,25 @@ for name,pkg in packages.iteritems():
 
 deph,makedeph = [],[]
 
-print "==> checking dependencies"
+print("==> checking dependencies")
 for name,pkg in repopkgs.iteritems():
 	(deps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.deps)
 	pkgdeps[pkg] = deps
 	missing_deps.extend(missdeps)
 	deph.extend(hierarchy)
 
-print "==> checking makedepends"
+print("==> checking makedepends")
 for name,pkg in repopkgs.iteritems():
 	(makedeps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.makedeps)
 	makepkgdeps[pkg] = makedeps
 	missing_makedeps.extend(missdeps)
 	makedeph.extend(hierarchy)
 
-print "==> checking hierarchy"
+print("==> checking hierarchy")
 dep_hierarchy = check_hierarchy(deph)
 makedep_hierarchy = check_hierarchy(makedeph)
 
-print "==> checking for circular dependencies"
+print("==> checking for circular dependencies")
 # make sure pkgdeps is filled for every package
 for name,pkg in packages.iteritems():
 	if pkg not in pkgdeps:
@@ -487,7 +487,7 @@ for name,pkg in packages.iteritems():
 		pkgdeps[pkg] = deps
 find_scc(repopkgs.values())
 
-print "==> checking for differences between db files and pkgbuilds"
+print("==> checking for differences between db files and pkgbuilds")
 for repo in repos:
 	for pkg in dbpkgs[repo]:
 		if not (pkg in repopkgs and repopkgs[pkg].repo == repo):
-- 
cgit v1.2.3-2-g168b


From d0222cb73c50fd23abd572e080b5b5231cb6d23b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9my=20Oudompheng?= <oudomphe@math.unice.fr>
Date: Sat, 4 Dec 2010 18:20:39 +0100
Subject: Add support for the [multilib] repo.

Signed-off-by: Pierre Schmitz <pierre@archlinux.de>
---
 cron-jobs/check_archlinux/check_packages.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'cron-jobs/check_archlinux')

diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py
index 948a61a..86494b7 100755
--- a/cron-jobs/check_archlinux/check_packages.py
+++ b/cron-jobs/check_archlinux/check_packages.py
@@ -244,7 +244,8 @@ def check_hierarchy(deph):
 def get_repo_hierarchy(repo):
 	repo_hierarchy = {'core': ['core'], \
 		'extra': ['core', 'extra'], \
-		'community': ['core', 'extra', 'community']}
+		'community': ['core', 'extra', 'community'], \
+                'multilib': ['core', 'extra', 'community', 'multilib'] }
 	if repo in repo_hierarchy:
 		return repo_hierarchy[repo]
 	else:
-- 
cgit v1.2.3-2-g168b


From bff3dabaf45abbc7d6bed1b3ec1c717ae678c8b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9my=20Oudompheng?= <oudomphe@math.unice.fr>
Date: Sat, 4 Dec 2010 18:37:17 +0100
Subject: Only emit a warning when a repository does not exist.

Signed-off-by: Pierre Schmitz <pierre@archlinux.de>
---
 cron-jobs/check_archlinux/check_packages.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'cron-jobs/check_archlinux')

diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py
index 86494b7..bdbbeed 100755
--- a/cron-jobs/check_archlinux/check_packages.py
+++ b/cron-jobs/check_archlinux/check_packages.py
@@ -403,8 +403,8 @@ for absroot in absroots:
 	for repo in repos:
 		repopath = absroot + "/" + repo
 		if not os.path.isdir(repopath):
-			print "Error : the repository " + repo + " does not exist in " + absroot
-			sys.exit()
+			print("Warning : the repository " + repo + " does not exist in " + absroot)
+
 if not os.path.isdir(repodir):
 	print "Error: the repository directory %s does not exist" % repodir
 	sys.exit()
-- 
cgit v1.2.3-2-g168b


From 76256c33bb6fa06b79bdab6615f2da587cd80f31 Mon Sep 17 00:00:00 2001
From: Pierre Schmitz <pierre@archlinux.de>
Date: Thu, 9 Dec 2010 08:34:10 +0100
Subject: check_packages.py: remove obsolete klibc workaround

---
 cron-jobs/check_archlinux/check_packages.py | 6 ------
 1 file changed, 6 deletions(-)

(limited to 'cron-jobs/check_archlinux')

diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py
index bdbbeed..0a2b10e 100755
--- a/cron-jobs/check_archlinux/check_packages.py
+++ b/cron-jobs/check_archlinux/check_packages.py
@@ -454,12 +454,6 @@ for name,pkg in repopkgs.iteritems():
 	archs = verify_archs(name,pkg.repo,pkg.archs)
 	invalid_archs.extend(archs)
 
-# ugly hack to strip the weird kblic- deps
-for name,pkg in packages.iteritems():
-	p = re.compile('klibc-[\w\-]{27}|klibc-\*')
-	pkg.deps = [dep for dep in pkg.deps if not p.match(dep)]
-	pkg.makedeps = [dep for dep in pkg.makedeps if not p.match(dep)]
-
 deph,makedeph = [],[]
 
 print("==> checking dependencies")
-- 
cgit v1.2.3-2-g168b