summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xcron-jobs/check_archlinux.py704
-rw-r--r--cron-jobs/check_archlinux/README8
-rw-r--r--cron-jobs/check_archlinux/alpm.c40
-rwxr-xr-xcron-jobs/check_archlinux/check_packages.py392
-rwxr-xr-xcron-jobs/check_archlinux/parse_pkgbuilds.sh78
-rw-r--r--cron-jobs/check_archlinux/setup.py10
6 files changed, 528 insertions, 704 deletions
diff --git a/cron-jobs/check_archlinux.py b/cron-jobs/check_archlinux.py
deleted file mode 100755
index cb43351..0000000
--- a/cron-jobs/check_archlinux.py
+++ /dev/null
@@ -1,704 +0,0 @@
-#!/usr/bin/env python
-#
-# check_archlinux.py
-#
-# Written by Scott Horowitz <stonecrest@gmail.com>
-# Graphical dependency tree by Cesar G. Miguel <cesargm@ime.usp.br>
-#
-# This script currently checks for a number of issues in your ABS tree:
-# 1. Directories with missing PKGBUILDS
-# 2. Duplicate PKGBUILDs
-# 3. Missing (make-)dependencies, taking into account 'provides'
-# 4. Provisioned dependencies
-# 5. Circular dependencies
-# 6. Valid arch's in PKGBUILDS
-# 7. Missing packages in Arch repos
-# 8. PKGBUILD names that don't match their directory
-# 9. Hierarchy of repos (e.g., that a core package doesn't depend on
-# a non-core package)
-# It can also, optionally, generate a graphical representation of the
-# dependency tree.
-#
-# Todo:
-# 1. Accommodate testing repo?
-
-abs_conf_dir = "/etc/abs"
-valid_archs = ['i686', 'x86_64']
-cvs_tags = {'i686': 'CURRENT', 'x86_64': 'CURRENT-64'}
-#include_paths = ['core', 'extra', 'community', 'unstable']
-include_paths = ['core', 'extra', 'unstable']
-pkgdir_path_depth = 3
-
-base_server = "ftp.archlinux.org" # Must be ftp site
-# Ensure that core repo only depends on core, and the extra repo only
-# depends on core and extra, etc.
-repo_hierarchy = {'core': ('core'), \
-'extra': ('core', 'extra'), \
-'unstable': ('core', 'extra', 'unstable'), \
-'community': ('core', 'extra', 'unstable', 'community')}
-
-pkgname_str = "pkgname="
-dep_str = "depends=("
-makedep_str = "makedepends=("
-provides_str = "provides=("
-arch_str = "arch=("
-pkgver_str = "pkgver="
-pkgrel_str = "pkgrel="
-build_str = "build()"
-source_str = "source="
-url_str = "url="
-mathlist = (">=", "<=", "=", ">", "<")
-sup_tag_str = "*default tag="
-
-import os, sys, getopt, tempfile
-from ftplib import FTP
-try:
- import pydot
- HAS_PYDOT = True
-except:
- HAS_PYDOT = False
-
-def print_heading(heading):
- print ""
- print "=" * (len(heading) + 4)
- print "= " + heading + " ="
- print "=" * (len(heading) + 4)
-
-def print_subheading(subheading):
- print ""
- print subheading
- print "-" * (len(subheading) + 2)
-
-def rmgeneric(path, __func__):
- try:
- __func__(path)
- except OSError, (errno, strerror):
- pass
-
-def removeall(path):
- if not os.path.isdir(path):
- return
-
- files=os.listdir(path)
-
- for x in files:
- fullpath=os.path.join(path, x)
- if os.path.isfile(fullpath):
- f=os.remove
- rmgeneric(fullpath, f)
- elif os.path.isdir(fullpath):
- removeall(fullpath)
- f=os.rmdir
- rmgeneric(fullpath, f)
-
-def update_var(line, user_vars, pkgpath):
- if line.count("$") > 0:
- export_line = ""
- for var in user_vars:
- if line.count(var[0]) > 0:
- export_line = export_line + var[0] + "=" + var[1] + " && "
- if line.startswith("(") and line.endswith(")"):
- line = line[1:-1]
- export_line = export_line + "echo \"" + line + "\""
- line = os.popen(export_line).read().replace("\n", "")
- return line
-
-def split_dep_prov_symbol(dep):
- # Splits 'foo>=1.2.3' into ('foo', '1.2.3', '>=')
- prov = ""
- symbol = ""
- for char in mathlist:
- pos = dep.find(char)
- if pos > -1:
- prov = dep[pos:].replace(char, "")
- dep = dep[:pos]
- symbol = char
- break
- return (dep, prov, symbol)
-
-def repo_from_path(path):
- # Community HACK: community still has old
- # community/category/pkgname/PKGBUILD path - accomodate for this
- if path.split("/")[-1 * (pkgdir_path_depth + 1)] == "community":
- return path.split("/")[-1 * (pkgdir_path_depth + 1)]
- return path.split("/")[-1 * pkgdir_path_depth]
-
-def create_supfile(sourcefile, destfile, newtag):
- o = open(sourcefile, 'r')
- info = o.read()
- o.close()
- lines = info.split("\n")
-
- o = open(destfile, 'w')
- for line in lines:
- line = line.strip()
- if line[:len(sup_tag_str)] == sup_tag_str:
- line = sup_tag_str + newtag
- o.write(line + "\n")
- o.close()
-
-def get_deps_provides_etc(pkgpath):
- # Parse PKGBUILD for name, depends, makedepends, provides, arch's, and version
- o = open(pkgpath, 'r')
- info = o.read()
- o.close()
- lines = info.split("\n")
-
- deps = []
- provides = []
- archs = []
- makedeps = []
- array = []
- user_vars = []
- continue_line = False
- for line in lines:
- line = line.strip()
- if line.find("#") > -1:
- line = line[:line.find("#")].strip()
- if not continue_line:
- deps_line = False
- provides_line = False
- arch_line = False
- makedeps_line = False
- if line[:len(dep_str)] == dep_str:
- line = line.replace(dep_str,"")
- deps_line = True
- elif line[:len(makedep_str)] == makedep_str:
- line = line.replace(makedep_str,"")
- makedeps_line = True
- elif line[:len(provides_str)] == provides_str:
- line = line.replace(provides_str,"")
- provides_line = True
- elif line[:len(arch_str)] == arch_str:
- line = line.replace(arch_str, "")
- arch_line = True
- elif line[:len(pkgname_str)] == pkgname_str:
- pkgname = line.replace(pkgname_str, "")
- if pkgname.startswith("\"") and pkgname.endswith("\""):
- pkgname = pkgname[1:-1]
- pkgname = update_var(pkgname, user_vars, pkgpath)
- user_vars.append([pkgname_str, pkgname])
- line = ""
- elif line[:len(pkgver_str)] == pkgver_str:
- pkgver = line.replace(pkgver_str, "")
- if pkgver.startswith("\"") and pkgver.endswith("\""):
- pkgver = pkgver[1:-1]
- pkgver = update_var(pkgver, user_vars, pkgpath)
- user_vars.append([pkgver_str[:-1], pkgver])
- line = ""
- elif line[:len(pkgrel_str)] == pkgrel_str:
- pkgrel = line.replace(pkgrel_str, "")
- if pkgrel.startswith("\"") and pkgrel.endswith("\""):
- pkgrel = pkgrel[1:-1]
- pkgrel = update_var(pkgrel, user_vars, pkgpath)
- user_vars.append([pkgrel_str[:-1], pkgrel])
- line = ""
- elif line[:len(build_str)] == build_str:
- break
- elif not continue_line:
- if line.count("=") == 1 and line.count(" ") == 0 and line[:1] != "#" and \
- line[:len(source_str)] != source_str and line[:len(url_str)] != url_str:
- split = line.split("=")
- for item in range(len(split)):
- split[item] = update_var(split[item], user_vars, pkgpath)
- user_vars.append(split)
- line = ""
- if len(line) > 0:
- pos = line.find(")")
- if pos > -1:
- # strip everything from closing paranthesis on
- # since some PKGBUILDS have comments after the
- # depends array
- line = line[:pos]
- line = line.split(' ')
- for i in range(len(line)):
- line[i] = line[i].replace("'","").replace('"','')
- line[i] = update_var(line[i], user_vars, pkgpath)
- if len(line[i]) > 0:
- if deps_line:
- deps.append(line[i])
- array=deps
- elif provides_line:
- provides.append(line[i])
- array=provides
- elif arch_line:
- archs.append(line[i])
- array=archs
- elif makedeps_line:
- makedeps.append(line[i])
- array=makedeps
- if array and (array[-1] == "\\" or array[-1][-1] == "\\"):
- # continue reading deps/provides on next line
- if array[-1] == "\\":
- array.pop(-1)
- else:
- array[-1] = array[-1].replace("\\", "")
- continue_line = True
- else:
- continue_line = False
- version = pkgver + "-" + pkgrel
- return (pkgname, deps, makedeps, provides, archs, version)
-
-def get_pkgbuilds_in_dir(rootdir):
- # Recursively populates pkgbuild_deps, pkgbuild_paths, etc.
- # dicts with info from each PKGBUILD found in rootdir:
- if rootdir != absroot:
- if rootdir.count("/") == (absroot.count("/")+1) and rootdir not in curr_include_paths:
- return
- pkgfound = False
- for f in os.listdir(rootdir):
- fpath = rootdir + "/" + f
- if os.path.isdir(fpath):
- get_pkgbuilds_in_dir(fpath)
- elif f == 'PKGBUILD':
- pkgfound = True
- name = rootdir.split("/")[-1]
- if name in pkgbuild_deps:
- dups.append(fpath.replace(absroot, "") + " vs. " + pkgbuild_paths[name].replace(absroot, ""))
- else:
- (pkgname, deps, makedeps, provides, archs, version) = get_deps_provides_etc(fpath)
- pkgbuild_deps[pkgname] = deps
- pkgbuild_makedeps[pkgname] = makedeps
- pkgbuild_paths[pkgname] = fpath
- pkgbuild_archs[pkgname] = archs
- pkgbuild_versions[pkgname] = version
- # We'll store the provides "backwards" compared to
- # the other dicts. This will make searching for
- # provides easier by being able to look up the
- # provide name itself and find what provides it
- for provide in provides:
- pkgbuild_provides[provide] = pkgname
- if pkgname != name:
- mismatches.append(pkgname + " vs. " + fpath.replace(absroot, ""))
- if not pkgfound and rootdir != absroot:
- repo = rootdir.replace(absroot, "").split("/")[1]
- num_slashes = pkgdir_path_depth - 1
- # Community HACK: community still has old
- # community/category/pkgname/PKGBUILD path - accomodate for this
- if repo == "community" and rootdir.replace(absroot, "").count("/") == num_slashes + 1 and rootdir.split("/")[-1] != "CVS":
- misses.append(rootdir.replace(absroot, "") + "/PKGBUILD")
- if repo != "community" and rootdir.replace(absroot, "").count("/") == num_slashes:
- misses.append(rootdir.replace(absroot, "") + "/PKGBUILD")
-
-def verify_depends_makedepends(verify_makedeps=False):
- # Make sure all the deps we parsed are actually packages; also
- # ensure we meet dep provisions.
- if verify_makedeps:
- array = pkgbuild_makedeps
- else:
- array = pkgbuild_deps
- for pkgname in array:
- deps = array[pkgname]
- pkg_repo = repo_from_path(pkgbuild_paths[pkgname])
- deps_to_pop = []
- for i in range(len(deps)):
- dep = deps[i]
- (dep, prov, char) = split_dep_prov_symbol(dep)
- try:
- x = pkgbuild_deps[dep]
- # Check that prov is met too:
- if len(prov) > 0:
- compare_str = "vercmp " + pkgbuild_versions[dep] + " " + prov
- error = False
- if char == "<=":
- if int(os.popen(compare_str).read().replace("\n", "")) > 0:
- error = True
- elif char == ">=":
- if int(os.popen(compare_str).read().replace("\n", "")) < 0:
- error = True
- elif char == "=":
- if int(os.popen(compare_str).read().replace("\n", "")) != 0:
- error = True
- elif char == ">":
- if int(os.popen(compare_str).read().replace("\n", "")) <= 0:
- error = True
- elif char == "<":
- if int(os.popen(compare_str).read().replace("\n", "")) >= 0:
- error = True
- if error:
- if verify_makedeps:
- unmet_makedep_provs.append(pkgname + " --> '" + dep + char + prov + "'")
- else:
- unmet_dep_provs.append(pkgname + " --> '" + dep + char + prov + "'")
- # Check repos fit hierarchy scheme:
- dep_repo = repo_from_path(pkgbuild_paths[dep])
- try:
- valid_repos = repo_hierarchy[pkg_repo]
- # Make sure dep repo is one of the valid repos:
- if dep_repo not in valid_repos:
- if verify_makedeps:
- makedep_hierarchy.append(pkg_repo + "/" + pkgname + " depends on " + dep_repo + "/" + dep)
- else:
- dep_hierarchy.append(pkg_repo + "/" + pkgname + " depends on " + dep_repo + "/" + dep)
- except:
- pass
- except:
- # Check if a package provides this dep:
- try:
- x = pkgbuild_provides[dep]
- except:
- if verify_makedeps:
- missing_makedeps.append(pkgname + " --> '" + dep + "'")
- else:
- missing_deps.append(pkgname + " --> '" + dep + "'")
- deps_to_pop.append(i)
- # Pop deps not found from end to beginning:
- while len(deps_to_pop) > 0:
- deps.pop(deps_to_pop[-1])
- deps_to_pop.pop(-1)
-
-def verify_archs():
- for pkgname in pkgbuild_archs:
- newarch = []
- archs = pkgbuild_archs[pkgname]
- for arch in archs:
- if arch not in valid_archs:
- invalid_archs.append(pkgname + " --> " + arch)
- else:
- newarch.append(arch)
- if len(newarch) > 0:
- pkgbuild_archs[pkgname] = newarch
-
-def verify_packages(tree_arch):
- # Make sure packages exist in Arch repo(s):
- ftp = FTP(base_server)
- ftp.login()
- prev_wd = ""
- # Find all repos/archs; marching through them in order will greatly speed
- # up searching for ftp files by minimizing the number of .nlst's that we
- # have to do.
- repos = []
- for pkgname in pkgbuild_paths:
- pkgrepo = repo_from_path(pkgbuild_paths[pkgname])
- if not pkgrepo in repos:
- repos.append(pkgrepo)
- archs = []
- for pkgname in pkgbuild_archs:
- pkgarchs = pkgbuild_archs[pkgname]
- for arch in pkgarchs:
- if not arch in archs:
- archs.append(arch)
- for r in repos:
- for pkgname in pkgbuild_archs:
- repo = repo_from_path(pkgbuild_paths[pkgname])
- if repo == r:
- archs = pkgbuild_archs[pkgname]
- pkgver_rel = pkgbuild_versions[pkgname]
- for arch in archs:
- if arch == tree_arch:
- # Check for file:
- wd = repo + "/os/" + arch
- if wd != prev_wd:
- ftpfiles = ftp.nlst(wd)
- prev_wd = wd
- fname_new = wd + "/" + pkgname + "-" + pkgver_rel + "-" + arch + ".pkg.tar.gz"
- fname_old = wd + "/" + pkgname + "-" + pkgver_rel + ".pkg.tar.gz"
- if fname_old not in ftpfiles and fname_new not in ftpfiles:
- missing_pkgs.append(pkgname + "-" + pkgver_rel + " in " + wd)
- ftp.quit()
-
-def subset_superset_path(currpath, currpathnum, paths):
- # If a pair of subset/superset paths are found,
- # pop the superset one to show the more minimal
- # case.
- #
- # e.g. foo > bar > baz > foo (superset)
- # foo > bar > foo (subset)
- # --> pop the superset
- #
- currdeps = currpath.split(">")
- currdeps = list(set(currdeps))
- currdeps.sort()
- pathnum = 0
- for path in paths:
- if pathnum != currpathnum:
- deps = path.split(">")
- deps = list(set(deps))
- deps.sort()
- if len(currdeps) < len(path):
- subset = True
- for d in currdeps:
- if not d in path:
- subset = False
- break
- if subset:
- circular_deps.pop(pathnum)
- if pathnum <= currpathnum:
- currpathnum -= 1
- elif len(currdeps) > len(path):
- superset = True
- for d in path:
- if not d in currdeps:
- superset = False
- break
- if superset:
- circular_deps.pop(currpathnum)
- currpathnum -= 1
- pathnum += 1
- return True
-
-def unique_path(currpath, paths):
- # Returns false if an equivalent path exists in
- # paths.
- #
- # e.g. foo > bar > foo
- # bar > foo > bar
- #
- currdeps = currpath.split(">")
- currdeps = list(set(currdeps))
- currdeps.sort()
- for path in paths:
- deps = path.split(">")
- deps = list(set(deps))
- deps.sort()
- if currdeps == deps:
- return False
- return True
-
-def update_paths(paths, dep, dep2):
- # Update paths by appending new paths with dep2
- # based on all the different ways we could get
- # to dep2. Returns True if a path was updated.
- new_path = False
- for i in range(len(paths)):
- array = paths[i].split(">")
- newpath = paths[i] + ">" + dep2
- if array[-1] == dep and not dep2 in array and unique_path(newpath, paths):
- paths.append(newpath)
- new_path = True
- return new_path
-
-def check_name_in_recursive_deps(pkgname):
- # Retrieve all recursive dependencies from a package and
- # determines if pkgname is found in one of its deps.
- recursive_deps = []
- for dep in pkgbuild_deps[pkgname]:
- dep = split_dep_prov_symbol(dep)[0] # Strip any provision
- recursive_deps.append(dep)
- paths = []
- for dep in recursive_deps:
- dep = split_dep_prov_symbol(dep)[0] # Strip any provision
- paths.append(dep)
- searching = True
- while searching:
- searching = False
- for dep in recursive_deps:
- for dep2 in pkgbuild_deps[dep]:
- dep2 = split_dep_prov_symbol(dep2)[0] # Strip any provision
- # This is a HUGE time-saver. Instead of generating every single
- # possible path that can yield a circular dep, we'll reduce
- # the number greatly by throwing out paths like such:
- #
- # If we have a path: foo>bar>baz>blah>poop>foo
- # We will throw out any:
- # foo>...>bar>baz>blah>poop>foo
- # foo>...>baz>blah>poop>foo
- # foo>...>blah>poop>foo
- # and so on. Otherwise we will find hundreds or even thousands
- # of possible paths that all essentially represent the same
- # circular dep.
- #
- # However, we will always let pkgname through in order to make
- # sure we can find multiple circular deps for a given pkg.
- if dep2 not in recursive_deps or dep2 == pkgname:
- updated = update_paths(paths, dep, dep2)
- if dep2 not in recursive_deps:
- recursive_deps.append(dep2)
- if updated:
- searching = True
- # Done searching, store circular deps:
- for path in paths:
- if path.split(">")[-1] == pkgname:
- if unique_path(pkgname + ">" + path, circular_deps):
- circular_deps.append(pkgname + ">" + path)
- # Reduce any subset/superset path pairs:
- pathnum = 0
- for path in circular_deps:
- subset_superset_path(path, pathnum, circular_deps)
- pathnum += 1
-
-def circular_deps_check():
- # Check for circular dependencies:
- for pkgname in pkgbuild_deps:
- check_name_in_recursive_deps(pkgname)
-
-def visualize_repo():
- output = 'digraph G { \n \
- concentrate = true; \n \
- ordering = out; \n \
- ranksep=5.0; \n \
- node [style=filled,fontsize=8]; \n'
-
- # draws circular dependencies in red
- for path in circular_deps:
- output += '\t "'+path[0]+'"'
- deps = path.split(">")
- for d in deps:
- output += ' -> "'+d+'"'
- output += ' [color=red]\n'
-
- for pkg in pkgbuild_deps.keys():
- output += '\t "'+pkg+'" -> { '
- for d in pkgbuild_deps[pkg]:
- d = split_dep_prov_symbol(d)[0] # Strip any provision
- output += '"'+d+'"; '
- output += '}\n'
-
- output += '}'
-
- # Uncomment these lines to get a file dump called
- # 'output'. This can be used to manually generate
- # an image using, e.g., dot -Tsvg output tree.svg
- #dump = open('output', 'w')
- #dump.write(output)
- #dump.close()
-
- fname = 'dependency_tree-' + arch + '.svg'
- print "Generating " + fname + "..."
- g = pydot.graph_from_dot_data(output)
- g.write(fname, prog='dot', format='svg')
-
-def print_result(list, subheading):
- if len(list) > 0:
- print_subheading(subheading)
- for item in list:
- print item
-
-def print_results():
- print_result(misses, "Missing PKGBUILDs")
- print_result(mismatches, "Mismatched Pkgnames")
- print_result(dups, "Duplicate PKGBUILDs")
- print_result(missing_deps, "Missing Dependencies")
- print_result(missing_makedeps, "Missing Makedepends")
- print_result(unmet_dep_provs, "Unmet Dependency Provisions")
- print_result(unmet_makedep_provs, "Unmet Makedepends Provisions")
- print_result(dep_hierarchy, "Repo Hierarchy for Dependencies")
- print_result(makedep_hierarchy, "Repo Hierarchy for Makedepends")
- print_result(invalid_archs, "Invalid Archs")
- print_result(circular_deps, "Circular Dependencies")
- print_result(missing_pkgs, "Missing Repo Packages")
- print_subheading("Summary")
- print "Dirs with missing PKGBUILDs: ", len(misses)
- print "Duplicate PKGBUILDs: ", len(dups)
- print "Missing (make)dependencies: ", len(missing_deps)+len(missing_makedeps)
- print "Unmet provisioned (make)dependencies: ", len(unmet_dep_provs)+len(unmet_makedep_provs)
- print "Circular dependencies: ", len(circular_deps)
- print "Invalid archs: ", len(invalid_archs)
- print "Missing packages in repos: ", len(missing_pkgs)
- print "Mismatching PKGBUILD names: ", len(mismatches)
- print "Repo hierarchy problems: ", len(dep_hierarchy)+len(makedep_hierarchy)
- print ""
-
-def print_usage():
- print ""
- print "Usage: check_archlinux [OPTION]"
- print ""
- print "Options:"
- print " --abs-tree=<path> REQUIRED Check specified tree (assumes the abs tree"
- print " is i686 unless overridden with --arch)"
- print " --arch=<arch> OPTIONAL Use specified arch (e.g. 'x86_64')"
- print " -g OPTIONAL Generate graphical dependency tree(s)"
- print " -h, --help OPTIONAL Show this help and exit"
- print ""
- print "Examples:"
- print "\n Check existing i686 abs tree:"
- print " check_archlinux --abs-tree=/var/abs"
- print "\n Check existing x86_64 abs tree and also generate dep tree image:"
- print " check_archlinux --abs-tree=/var/abs --arch=x86_64 -g"
- print ""
-
-graphdeptree = False
-user_absroot = ""
-user_arch = ""
-try:
- opts, args = getopt.getopt(sys.argv[1:], "g", ["abs-tree=", "arch="])
-except getopt.GetoptError:
- print_usage()
- sys.exit()
-if opts != []:
- for o, a in opts:
- if o in ("-g"):
- graphdeptree = True
- if not HAS_PYDOT:
- print "You must install pydot to generate a graphical dependency tree. Aborting..."
- sys.exit()
- elif o in ("--abs-tree"):
- user_absroot = a
- elif o in ("--arch"):
- user_arch = a
- if user_arch not in valid_archs:
- print "You did not specify a valid arch. Aborting..."
- sys.exit()
- else:
- print_usage()
- sys.exit()
- if args != []:
- for a in args:
- if a in ("play", "pause", "stop", "next", "prev", "pp", "info", "status", "repeat", "shuffle"):
- self.single_connect_for_passed_arg(a)
- else:
- print_usage()
- sys.exit()
-
-if len(user_absroot) == 0:
- print_usage()
- sys.exit()
-
-if len(user_arch) == 0:
- user_arch = valid_archs[0] # i686 default..
-
-if len(user_absroot) > 0:
- print "Warning: Ensure your ABS tree is clean to prevent false positives."
-
-try:
- for arch in valid_archs:
- if len(user_arch) == 0 or user_arch == arch:
- print_heading(arch + " Integrity Check")
- absroot = user_absroot
- curr_include_paths = []
- for repo in include_paths:
- curr_include_paths.append(absroot + "/" + repo)
-
- # Re-init vars for new abs tree:
- pkgbuild_deps = {} # pkgname: [dep1, dep2>foo, dep3=bar, ...]
- pkgbuild_makedeps = {} # pkgname: [dep1, dep2>foo, dep3=bar, ...]
- pkgbuild_provides = {} # provide_name: pkgname
- pkgbuild_paths = {} # pkgname: /var/abs/foo/bar/pkgname/PKGBUILD
- pkgbuild_archs = {} # pkgname: [i686, x86_64]
- pkgbuild_versions = {} # pkname: 1.0.4-2
- # circular_deps is not a dict to accommodate multiple circ deps for a given pkg
- circular_deps = [] # pkgname>dep1>dep2>...>pkgname
- mismatches = []
- misses = []
- dups = []
- missing_deps = []
- unmet_dep_provs = []
- dep_hierarchy = []
- missing_makedeps = []
- unmet_makedep_provs = []
- makedep_hierarchy = []
- invalid_archs = []
- missing_pkgs = []
-
- # Verify stuff for abs tree:
- print "\nPerforming integrity checks..."
- print "==> parsing pkgbuilds"
- get_pkgbuilds_in_dir(absroot)
- print "==> checking dependencies"
- verify_depends_makedepends()
- print "==> checking makedepends"
- verify_depends_makedepends(True)
- print "==> checking archs"
- verify_archs()
- print "==> checking for circular dependencies"
- circular_deps_check()
- print "==> checking repo packages"
- verify_packages(arch)
- print_results()
- if graphdeptree:
- visualize_repo()
-
-except:
- sys.exit()
-
-# vim: set ts=2 sw=2 noet :
diff --git a/cron-jobs/check_archlinux/README b/cron-jobs/check_archlinux/README
new file mode 100644
index 0000000..f3a1b90
--- /dev/null
+++ b/cron-jobs/check_archlinux/README
@@ -0,0 +1,8 @@
+1) Build the python module
+$ python setup.py build
+
+2) copy it back to the current working directory
+$ cp build/lib.*/alpm.* .
+
+3) run the script
+$ ./check_packages.py -h
diff --git a/cron-jobs/check_archlinux/alpm.c b/cron-jobs/check_archlinux/alpm.c
new file mode 100644
index 0000000..0b7cd2c
--- /dev/null
+++ b/cron-jobs/check_archlinux/alpm.c
@@ -0,0 +1,40 @@
+#include <Python.h>
+#include <alpm.h>
+
+static PyObject *
+alpm_vercmp(PyObject *self, PyObject *args)
+{
+ const char *v1, *v2;
+ int ret;
+
+ if (!PyArg_ParseTuple(args, "ss", &v1, &v2))
+ return NULL;
+ ret = alpm_pkg_vercmp(v1, v2);
+ return Py_BuildValue("i", ret);
+}
+
+static PyMethodDef AlpmMethods[] = {
+ {"vercmp", alpm_vercmp, METH_VARARGS,
+ "Execute vercmp."},
+ {NULL, NULL, 0, NULL} /* Sentinel */
+};
+
+PyMODINIT_FUNC
+initalpm(void)
+{
+ (void) Py_InitModule("alpm", AlpmMethods);
+}
+
+int
+main(int argc, char *argv[])
+{
+ /* Pass argv[0] to the Python interpreter */
+ Py_SetProgramName(argv[0]);
+
+ /* Initialize the Python interpreter. Required. */
+ Py_Initialize();
+
+ /* Add a static module */
+ initalpm();
+ return 0;
+}
diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py
new file mode 100755
index 0000000..2d2efbe
--- /dev/null
+++ b/cron-jobs/check_archlinux/check_packages.py
@@ -0,0 +1,392 @@
+#!/usr/bin/python
+#
+# check_archlinux.py
+#
+# Original script by Scott Horowitz <stonecrest@gmail.com>
+# Rewritten by Xavier Chantry <shiningxc@gmail.com>
+#
+# This script currently checks for a number of issues in your ABS tree:
+# 1. Directories with missing PKGBUILDS
+# 2. Invalid PKGBUILDs (bash syntax error for instance)
+# 3. PKGBUILD names that don't match their directory
+# 4. Duplicate PKGBUILDs
+# 5. Valid arch's in PKGBUILDS
+# 6. Missing (make-)dependencies
+# 7. Hierarchy of repos (e.g., that a core package doesn't depend on
+# a non-core package)
+# 8. Circular dependencies
+
+import os,re,commands,getopt,sys,alpm
+import pdb
+
+packages = {} # pkgname : PacmanPackage
+provisions = {} # provision : PacmanPackage
+pkgdeps,makepkgdeps = {},{} # pkgname : list of the PacmanPackage dependencies
+invalid_pkgbuilds = []
+missing_pkgbuilds = []
+dups = []
+
+mismatches = []
+missing_deps = []
+missing_makedeps = []
+invalid_archs = []
+dep_hierarchy = []
+makedep_hierarchy = []
+circular_deps = [] # pkgname>dep1>dep2>...>pkgname
+checked_deps = []
+
+class PacmanPackage:
+ def __init__(self):
+ self.name,self.version = "",""
+ self.path,self.repo = "",""
+ self.deps,self.makedeps = [],[]
+ self.provides,self.conflicts = [],[]
+ self.archs = []
+
+class Depend:
+ def __init__(self,name,version,mod):
+ self.name = name
+ self.version = version
+ self.mod = mod
+
+def parse_pkgbuilds(repos):
+ oldcwd = os.getcwd()
+ os.chdir(absroot)
+ for repo in repos:
+ data = commands.getoutput(oldcwd + '/parse_pkgbuilds.sh ' + repo)
+ parse_data(repo,data)
+ os.chdir(oldcwd)
+
+def parse_data(repo,data):
+ attrname = None
+
+ for line in data.split('\n'):
+ if line.startswith('%'):
+ attrname = line.strip('%').lower()
+ elif line.strip() == '':
+ attrname = None
+ elif attrname == "invalid":
+ if repo in repos:
+ invalid_pkgbuilds.append(line)
+ elif attrname == "missing":
+ if repo in repos:
+ missing_pkgbuilds.append(line)
+ elif attrname == "name":
+ pkg = PacmanPackage()
+ pkg.name = line
+ pkg.repo = repo
+ dup = None
+ if packages.has_key(pkg.name):
+ dup = packages[pkg.name]
+ packages[pkg.name] = pkg
+ elif attrname == "version":
+ pkg.version = line
+ elif attrname == "path":
+ pkg.path = line
+ if dup != None and (pkg.repo in repos or dup.repo in repos):
+ dups.append(pkg.path + " vs. " + dup.path)
+ elif attrname == "arch":
+ pkg.archs.append(line)
+ elif attrname == "depends":
+ pkg.deps.append(line)
+ elif attrname == "makedepends":
+ pkg.makedeps.append(line)
+ elif attrname == "conflicts":
+ pkg.conflicts.append(line)
+ elif attrname == "provides":
+ pkg.provides.append(line)
+ provname=line.split("=")[0]
+ if not provisions.has_key(provname):
+ provisions[provname] = []
+ provisions[provname].append(pkg)
+
+def splitdep(dep):
+ name = dep
+ version = ""
+ mod = ""
+ for char in (">=", "<=", "=", ">", "<"):
+ pos = dep.find(char)
+ if pos > -1:
+ name = dep[:pos]
+ version = dep[pos:].replace(char, "")
+ mod = char
+ break
+ return Depend(name,version,mod)
+
+def splitprov(prov):
+ name = prov
+ version = ""
+ pos = prov.find("=")
+ if pos > -1:
+ name = prov[:pos]
+ version = prov[pos:].replace("=", "")
+ return (name,version)
+
+def vercmp(v1,mod,v2):
+ res = alpm.vercmp(v1,v2)
+ if res == 0:
+ return (mod.find("=") > -1)
+ elif res < 0:
+ return (mod.find("<") > -1)
+ elif res > 0:
+ return (mod.find(">") > -1)
+ return False
+
+
+def depcmp(name,version,dep):
+ if name != dep.name:
+ return False
+ if dep.version == "" or dep.mod == "":
+ return True
+ if version == "":
+ return False
+ return vercmp(version,dep.mod,dep.version)
+
+def provcmp(pkg,dep):
+ for prov in pkg.provides:
+ (provname,provver) = splitprov(prov)
+ if depcmp(provname,provver,dep):
+ return True
+ return False
+
+def verify_dep(dep):
+ dep = splitdep(dep)
+ if packages.has_key(dep.name):
+ pkg = packages[dep.name]
+ if depcmp(pkg.name,pkg.version,dep):
+ return [pkg]
+ if provisions.has_key(dep.name):
+ provlist = provisions[dep.name]
+ results = []
+ for prov in provlist:
+ if provcmp(prov,dep):
+ results.append(prov)
+ return results
+ return []
+
+def verify_deps(name,repo,deps):
+ pkg_deps = []
+ missdeps = []
+ hierarchy = []
+ for dep in deps:
+ pkglist = verify_dep(dep)
+ if pkglist == []:
+ missdeps.append(name + " --> '" + dep + "'")
+ else:
+ valid_repos = get_repo_hierarchy(repo)
+ pkgdep = None
+ for pkg in pkglist:
+ if pkg.repo in valid_repos:
+ pkgdep = pkg
+ break
+ if not pkgdep:
+ pkgdep = pkglist[0]
+ hierarchy.append(repo + "/" + name + " depends on " + pkgdep.repo + "/" + pkgdep.name)
+ pkg_deps.append(pkgdep)
+
+ return (pkg_deps,missdeps,hierarchy)
+
+def get_repo_hierarchy(repo):
+ repo_hierarchy = {'core': ['core'], \
+ 'extra': ['core', 'extra'], \
+ 'community': ['core', 'extra', 'community']}
+ if repo_hierarchy.has_key(repo):
+ return repo_hierarchy[repo]
+ else:
+ return ['core','extra','community']
+
+def verify_archs(name,archs):
+ valid_archs = ['i686', 'x86_64']
+ invalid_archs = []
+ for arch in archs:
+ if arch not in valid_archs:
+ invalid_archs.append(name + " --> " + arch)
+ return invalid_archs
+
+def find_scc(packages):
+ # reset all variables
+ global index,S,pkgindex,pkglowlink
+ index = 0
+ S = []
+ pkgindex = {}
+ pkglowlink = {}
+ cycles = []
+ for pkg in packages:
+ tarjan(pkg)
+
+def tarjan(pkg):
+ global index,S,pkgindex,pkglowlink,cycles
+ pkgindex[pkg] = index
+ pkglowlink[pkg] = index
+ index += 1
+ checked_deps.append(pkg)
+ S.append(pkg)
+ if pkgdeps.has_key(pkg):
+ deps = pkgdeps[pkg]
+ else:
+ print pkg.name
+ deps = []
+ for dep in deps:
+ if not pkgindex.has_key(dep):
+ tarjan(dep)
+ pkglowlink[pkg] = min(pkglowlink[pkg],pkglowlink[dep])
+ elif dep in S:
+ pkglowlink[pkg] = min(pkglowlink[pkg],pkgindex[dep])
+ if pkglowlink[pkg] == pkgindex[pkg]:
+ dep = S.pop()
+ if pkg == dep:
+ return
+ path = pkg.name
+ while pkg != dep:
+ path = dep.name + ">" + path
+ dep = S.pop()
+ path = dep.name + ">" + path
+ if pkg.repo in repos:
+ circular_deps.append(path)
+
+def print_heading(heading):
+ print ""
+ print "=" * (len(heading) + 4)
+ print "= " + heading + " ="
+ print "=" * (len(heading) + 4)
+
+def print_subheading(subheading):
+ print ""
+ print subheading
+ print "-" * (len(subheading) + 2)
+
+def print_missdeps(pkgname,missdeps) :
+ for d in missdeps:
+ print pkgname + " : " + d
+
+def print_result(list, subheading):
+ if len(list) > 0:
+ print_subheading(subheading)
+ for item in list:
+ print item
+
+def print_results():
+ print_result(missing_pkgbuilds, "Missing PKGBUILDs")
+ print_result(invalid_pkgbuilds, "Invalid PKGBUILDs")
+ print_result(mismatches, "Mismatched Pkgnames")
+ print_result(dups, "Duplicate PKGBUILDs")
+ print_result(invalid_archs, "Invalid Archs")
+ print_result(missing_deps, "Missing Dependencies")
+ print_result(missing_makedeps, "Missing Makedepends")
+ print_result(dep_hierarchy, "Repo Hierarchy for Dependencies")
+ print_result(makedep_hierarchy, "Repo Hierarchy for Makedepends")
+ print_result(circular_deps, "Circular Dependencies")
+ print_subheading("Summary")
+ print "Missing PKGBUILDs: ", len(missing_pkgbuilds)
+ print "Invalid PKGBUILDs: ", len(invalid_pkgbuilds)
+ print "Mismatching PKGBUILD names: ", len(mismatches)
+ print "Duplicate PKGBUILDs: ", len(dups)
+ print "Invalid archs: ", len(invalid_archs)
+ print "Missing (make)dependencies: ", len(missing_deps)+len(missing_makedeps)
+ print "Repo hierarchy problems: ", len(dep_hierarchy)+len(makedep_hierarchy)
+ print "Circular dependencies: ", len(circular_deps)
+ print ""
+
+def print_usage():
+ print ""
+ print "Usage: ./check_packages.py [OPTION]"
+ print ""
+ print "Options:"
+ print " --abs-tree=<path> Check specified tree (default : /var/abs)"
+ print " --repos=<r1,r2,...> Check specified repos (default : core,extra)"
+ print " -h, --help Show this help and exit"
+ print ""
+ print "Examples:"
+ print "\n Check core and extra in existing abs tree:"
+ print " ./check_packages.py --abs-tree=/var/abs --repos=core,extra"
+ print "\n Check community:"
+ print " ./check_packages.py --abs-tree=/var/abs --repos=community"
+ print ""
+
+## Default path to the abs root directory
+absroot = "/var/abs"
+## Default list of repos to check
+repos = ['core', 'extra']
+
+try:
+ opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos="])
+except getopt.GetoptError:
+ print_usage()
+ sys.exit()
+if opts != []:
+ for o, a in opts:
+ if o in ("--abs-tree"):
+ absroot = a
+ elif o in ("--repos"):
+ repos = a.split(",")
+ else:
+ print_usage()
+ sys.exit()
+ if args != []:
+ print_usage()
+ sys.exit()
+
+if not os.path.isdir(absroot):
+ print "Error : the abs tree " + absroot + " does not exist"
+ sys.exit()
+for repo in repos:
+ repopath = absroot + "/" + repo
+ if not os.path.isdir(repopath):
+ print "Error : the repository " + repo + " does not exist in " + absroot
+ sys.exit()
+# repos which need to be loaded
+loadrepos = set([])
+for repo in repos:
+ loadrepos = loadrepos | set(get_repo_hierarchy(repo))
+
+print_heading("Integrity Check")
+print "\nPerforming integrity checks..."
+
+print "==> parsing pkgbuilds"
+parse_pkgbuilds(loadrepos)
+
+repopkgs = {}
+for name,pkg in packages.iteritems():
+ if pkg.repo in repos:
+ repopkgs[name] = pkg
+
+print "==> checking mismatches"
+for name,pkg in repopkgs.iteritems():
+ pkgdirname = pkg.path.split("/")[-1]
+ if name != pkgdirname:
+ mismatches.append(name + " vs. " + pkg.path)
+
+print "==> checking archs"
+for name,pkg in repopkgs.iteritems():
+ archs = verify_archs(name,pkg.archs)
+ invalid_archs.extend(archs)
+
+# ugly hack to strip the weird kblic- deps
+for name,pkg in packages.iteritems():
+ p = re.compile('klibc-[A-Za-z0-9]{20,}|klibc-\*')
+ pkg.deps = [dep for dep in pkg.deps if not p.match(dep)]
+ pkg.makedeps = [dep for dep in pkg.makedeps if not p.match(dep)]
+
+print "==> checking dependencies"
+for name,pkg in repopkgs.iteritems():
+ (deps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.deps)
+ pkgdeps[pkg] = deps
+ missing_deps.extend(missdeps)
+ dep_hierarchy.extend(hierarchy)
+
+print "==> checking makedepends"
+for name,pkg in repopkgs.iteritems():
+ (makedeps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.makedeps)
+ makepkgdeps[pkg] = makedeps
+ missing_makedeps.extend(missdeps)
+ makedep_hierarchy.extend(hierarchy)
+
+print "==> checking for circular dependencies"
+# make sure pkgdeps is filled for every package
+for name,pkg in packages.iteritems():
+ if not pkgdeps.has_key(pkg):
+ (deps,missdeps,_) = verify_deps(name,pkg.repo,pkg.deps)
+ pkgdeps[pkg] = deps
+find_scc(repopkgs.values())
+
+print_results()
diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh
new file mode 100755
index 0000000..d4205ae
--- /dev/null
+++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh
@@ -0,0 +1,78 @@
+#!/bin/bash
+
+parse() {
+ unset pkgname pkgver pkgrel
+ unset depends makedepends conflicts provides
+ ret=0
+ dir=$1
+ pkgbuild=$dir/PKGBUILD
+ source $pkgbuild &>/dev/null || ret=$?
+
+ # ensure $pkgname and $pkgver variables were found
+ if [ $ret -ne 0 -o -z "$pkgname" -o -z "$pkgver" ]; then
+ echo -e "%INVALID%\n$pkgbuild\n"
+ return 1
+ fi
+
+ echo -e "%NAME%\n$pkgname\n"
+ echo -e "%VERSION%\n$pkgver-$pkgrel\n"
+ echo -e "%PATH%\n$dir\n"
+
+ if [ -n "$arch" ]; then
+ echo "%ARCH%"
+ for i in ${arch[@]}; do echo $i; done
+ echo ""
+ fi
+ if [ -n "$depends" ]; then
+ echo "%DEPENDS%"
+ for i in ${depends[@]}; do
+ echo $i
+ done
+ echo ""
+ fi
+ if [ -n "$makedepends" ]; then
+ echo "%MAKEDEPENDS%"
+ for i in ${makedepends[@]}; do
+ echo $i
+ done
+ echo ""
+ fi
+ if [ -n "$conflicts" ]; then
+ echo "%CONFLICTS%"
+ for i in ${conflicts[@]}; do echo $i; done
+ echo ""
+ fi
+ if [ -n "$provides" ]; then
+ echo "%PROVIDES%"
+ for i in ${provides[@]}; do echo $i; done
+ echo ""
+ fi
+ return 0
+}
+
+find_pkgbuilds() {
+ if [ -f $1/PKGBUILD ]; then
+ parse $1
+ return
+ fi
+ empty=1
+ for dir in $1/*; do
+ if [ -d $dir ]; then
+ find_pkgbuilds $dir
+ unset empty
+ fi
+ done
+ if [ -n "$empty" ]; then
+ echo -e "%MISSING%\n$1\n"
+ fi
+}
+
+if [ -z "$*" ]; then
+ exit 1
+fi
+
+for dir in "$@"; do
+ find_pkgbuilds $dir
+done
+
+exit 0
diff --git a/cron-jobs/check_archlinux/setup.py b/cron-jobs/check_archlinux/setup.py
new file mode 100644
index 0000000..b172752
--- /dev/null
+++ b/cron-jobs/check_archlinux/setup.py
@@ -0,0 +1,10 @@
+from distutils.core import setup, Extension
+
+alpm = Extension('alpm',
+ libraries = ['alpm'],
+ sources = ['alpm.c'])
+
+setup (name = 'Alpm',
+ version = '1.0',
+ description = 'Alpm bindings',
+ ext_modules = [alpm])