diff options
author | Joshua Ismael Haase Hernández <hahj87@gmail.com> | 2011-04-16 00:44:41 -0500 |
---|---|---|
committer | Joshua Ismael Haase Hernández <hahj87@gmail.com> | 2011-04-16 00:44:41 -0500 |
commit | 7203e2e2631e29a79f4ea822c371d4e12c9c1a88 (patch) | |
tree | f0ada9061babb0f656afa90e438bba0f0f781241 /cron-jobs | |
parent | deab65fad4ced009fb31f7033b1db8ef0af78aee (diff) | |
parent | 229a9c504cbd733c93cf91399dc54bedf5160cc5 (diff) |
Merge branch 'master' of ~/packages/dbscripts into prueba
Conflicts:
.gitignore
db-functions
Diffstat (limited to 'cron-jobs')
-rw-r--r-- | cron-jobs/check_archlinux/README | 8 | ||||
-rw-r--r-- | cron-jobs/check_archlinux/alpm.c | 40 | ||||
-rwxr-xr-x | cron-jobs/check_archlinux/alpm.so | bin | 0 -> 7166 bytes | |||
-rwxr-xr-x | cron-jobs/check_archlinux/check_packages.py | 494 | ||||
-rwxr-xr-x | cron-jobs/check_archlinux/parse_pkgbuilds.sh | 149 | ||||
-rw-r--r-- | cron-jobs/check_archlinux/setup.py | 10 | ||||
-rwxr-xr-x | cron-jobs/create-filelists | 101 | ||||
-rwxr-xr-x | cron-jobs/devlist-mailer | 28 | ||||
-rwxr-xr-x | cron-jobs/ftpdir-cleanup | 113 | ||||
-rwxr-xr-x | cron-jobs/integrity-check | 32 | ||||
-rwxr-xr-x | cron-jobs/sourceballs | 150 | ||||
-rw-r--r-- | cron-jobs/sourceballs.force | 4 | ||||
-rw-r--r-- | cron-jobs/sourceballs.skip | 29 | ||||
-rwxr-xr-x | cron-jobs/sourceballs2 | 87 | ||||
-rwxr-xr-x | cron-jobs/update-web-db | 78 | ||||
l--------- | cron-jobs/update-web-files-db | 1 |
16 files changed, 1324 insertions, 0 deletions
diff --git a/cron-jobs/check_archlinux/README b/cron-jobs/check_archlinux/README new file mode 100644 index 0000000..8f178dc --- /dev/null +++ b/cron-jobs/check_archlinux/README @@ -0,0 +1,8 @@ +1) Build the python module +$ python2 setup.py build + +2) copy it back to the current working directory +$ cp build/lib.*/alpm.* . + +3) run the script +$ ./check_packages.py -h diff --git a/cron-jobs/check_archlinux/alpm.c b/cron-jobs/check_archlinux/alpm.c new file mode 100644 index 0000000..0b7cd2c --- /dev/null +++ b/cron-jobs/check_archlinux/alpm.c @@ -0,0 +1,40 @@ +#include <Python.h> +#include <alpm.h> + +static PyObject * +alpm_vercmp(PyObject *self, PyObject *args) +{ + const char *v1, *v2; + int ret; + + if (!PyArg_ParseTuple(args, "ss", &v1, &v2)) + return NULL; + ret = alpm_pkg_vercmp(v1, v2); + return Py_BuildValue("i", ret); +} + +static PyMethodDef AlpmMethods[] = { + {"vercmp", alpm_vercmp, METH_VARARGS, + "Execute vercmp."}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +PyMODINIT_FUNC +initalpm(void) +{ + (void) Py_InitModule("alpm", AlpmMethods); +} + +int +main(int argc, char *argv[]) +{ + /* Pass argv[0] to the Python interpreter */ + Py_SetProgramName(argv[0]); + + /* Initialize the Python interpreter. Required. */ + Py_Initialize(); + + /* Add a static module */ + initalpm(); + return 0; +} diff --git a/cron-jobs/check_archlinux/alpm.so b/cron-jobs/check_archlinux/alpm.so Binary files differnew file mode 100755 index 0000000..50f0f97 --- /dev/null +++ b/cron-jobs/check_archlinux/alpm.so diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py new file mode 100755 index 0000000..0a2b10e --- /dev/null +++ b/cron-jobs/check_archlinux/check_packages.py @@ -0,0 +1,494 @@ +#!/usr/bin/python2 +# +# check_archlinux.py +# +# Original script by Scott Horowitz <stonecrest@gmail.com> +# Rewritten by Xavier Chantry <shiningxc@gmail.com> +# +# This script currently checks for a number of issues in your ABS tree: +# 1. Directories with missing PKGBUILDS +# 2. Invalid PKGBUILDs (bash syntax error for instance) +# 3. PKGBUILD names that don't match their directory +# 4. Duplicate PKGBUILDs +# 5. Valid arch's in PKGBUILDS +# 6. Missing (make-)dependencies +# 7. Hierarchy of repos (e.g., that a core package doesn't depend on +# a non-core package) +# 8. Circular dependencies + +import os,re,commands,getopt,sys,tarfile,alpm +import pdb + +DBEXT='.db.tar.gz' + +packages = {} # pkgname : PacmanPackage +repopkgs = {} # pkgname : PacmanPackage +provisions = {} # provision : PacmanPackage +pkgdeps,makepkgdeps = {},{} # PacmanPackage : list of the PacmanPackage dependencies +invalid_pkgbuilds = [] +missing_pkgbuilds = [] +dups = [] + +dbonly = [] +absonly = [] + +mismatches = [] +missing_deps = [] +missing_makedeps = [] +invalid_archs = [] +dep_hierarchy = [] +makedep_hierarchy = [] +circular_deps = [] # pkgname>dep1>dep2>...>pkgname +checked_deps = [] + +class PacmanPackage: + def __init__(self): + self.name,self.version = "","" + self.base = "" + self.path,self.repo = "","" + self.deps,self.makedeps = [],[] + self.provides,self.conflicts = [],[] + self.archs = [] + +class Depend: + def __init__(self,name,version,mod): + self.name = name + self.version = version + self.mod = mod + +def parse_pkgbuilds(repos,arch): + for absroot in absroots: + for repo in repos: + cmd = os.path.dirname(os.path.realpath(sys.argv[0])) + '/parse_pkgbuilds.sh ' + cmd += arch + ' ' + absroot + '/' + repo + (status,output) = commands.getstatusoutput(cmd) + if status != 0: + print "Error : failed to run '%s'" % cmd + sys.exit() + parse_data(repo,output) + +def parse_data(repo,data): + attrname = None + + for line in data.split('\n'): + if line.startswith('%'): + attrname = line.strip('%').lower() + elif line.strip() == '': + attrname = None + elif attrname == "invalid": + if repo in repos: + invalid_pkgbuilds.append(line) + elif attrname == "missing": + if repo in repos: + missing_pkgbuilds.append(line) + elif attrname == "name": + pkg = PacmanPackage() + pkg.name = line + pkg.repo = repo + dup = None + if pkg.name in packages: + dup = packages[pkg.name] + else: + packages[pkg.name] = pkg + elif attrname == "base": + pkg.base = line + elif attrname == "version": + pkg.version = line + elif attrname == "path": + pkg.path = line + if dup != None and (pkg.repo in repos or dup.repo in repos): + dups.append(pkg.path + " vs. " + dup.path) + elif attrname == "arch": + pkg.archs.append(line) + elif attrname == "depends": + pkg.deps.append(line) + elif attrname == "makedepends": + pkg.makedeps.append(line) + elif attrname == "conflicts": + pkg.conflicts.append(line) + elif attrname == "provides": + pkg.provides.append(line) + +def parse_dbs(repos,arch): + dbpkgs = {} + for repo in repos: + pkgs = set([]) + db = tarfile.open(os.path.join(repodir,repo,'os',arch,repo + DBEXT)) + for line in db.getnames(): + if not '/' in line: + pkgs.add(line.rsplit('-',2)[0]) + dbpkgs[repo] = pkgs + return(dbpkgs) + +def splitdep(dep): + name = dep + version = "" + mod = "" + for char in (">=", "<=", "=", ">", "<"): + pos = dep.find(char) + if pos > -1: + name = dep[:pos] + version = dep[pos:].replace(char, "") + mod = char + break + return Depend(name,version,mod) + +def splitprov(prov): + name = prov + version = "" + pos = prov.find("=") + if pos > -1: + name = prov[:pos] + version = prov[pos:].replace("=", "") + return (name,version) + +def vercmp(v1,mod,v2): + res = alpm.vercmp(v1,v2) + if res == 0: + return (mod.find("=") > -1) + elif res < 0: + return (mod.find("<") > -1) + elif res > 0: + return (mod.find(">") > -1) + return False + + +def depcmp(name,version,dep): + if name != dep.name: + return False + if dep.version == "" or dep.mod == "": + return True + if version == "": + return False + return vercmp(version,dep.mod,dep.version) + +def provcmp(pkg,dep): + for prov in pkg.provides: + (provname,provver) = splitprov(prov) + if depcmp(provname,provver,dep): + return True + return False + +def verify_dep(dep): + dep = splitdep(dep) + if dep.name in packages: + pkg = packages[dep.name] + if depcmp(pkg.name,pkg.version,dep): + return [pkg] + if dep.name in provisions: + provlist = provisions[dep.name] + results = [] + for prov in provlist: + if provcmp(prov,dep): + results.append(prov) + return results + return [] + +def verify_deps(name,repo,deps): + pkg_deps = [] + missdeps = [] + hierarchy = [] + for dep in deps: + pkglist = verify_dep(dep) + if pkglist == []: + missdeps.append(repo + "/" + name + " --> '" + dep + "'") + else: + valid_repos = get_repo_hierarchy(repo) + pkgdep = None + for pkg in pkglist: + if pkg.repo in valid_repos: + pkgdep = pkg + break + if not pkgdep: + pkgdep = pkglist[0] + hierarchy.append((repo,name,pkgdep)) + + pkg_deps.append(pkgdep) + + return (pkg_deps,missdeps,hierarchy) + +def compute_deplist(pkg): + list = [] + stack = [pkg] + while stack != []: + dep = stack.pop() + if dep in pkgdeps: + for dep2 in pkgdeps[dep]: + if dep2 not in list: + list.append(dep2) + stack.append(dep2) + if dep in makepkgdeps: + for dep2 in makepkgdeps[dep]: + if dep2 not in list: + list.append(dep2) + stack.append(dep2) + return list + +def check_hierarchy(deph): + hierarchy = [] + for (repo,name,pkgdep) in deph: + deplist = compute_deplist(pkgdep) + valid_repos = get_repo_hierarchy(repo) + extdeps = [] + for dep in deplist: + if dep.repo not in valid_repos: + extdeps.append(dep.name) + string = repo + "/" + name + " depends on " + pkgdep.repo + "/" + pkgdep.name + " (" + string += "%s extra (make)deps to pull" % len(extdeps) + if 0 < len(extdeps) < 10: + string += " : " + ' '.join(extdeps) + string += ")" + hierarchy.append(string) + return hierarchy + +def get_repo_hierarchy(repo): + repo_hierarchy = {'core': ['core'], \ + 'extra': ['core', 'extra'], \ + 'community': ['core', 'extra', 'community'], \ + 'multilib': ['core', 'extra', 'community', 'multilib'] } + if repo in repo_hierarchy: + return repo_hierarchy[repo] + else: + return ['core','extra','community'] + +def verify_archs(name,repo,archs): + valid_archs = ['any', 'i686', 'x86_64'] + invalid_archs = [] + for arch in archs: + if arch not in valid_archs: + invalid_archs.append(repo + "/" + name + " --> " + arch) + return invalid_archs + +def find_scc(packages): + # reset all variables + global index,S,pkgindex,pkglowlink + index = 0 + S = [] + pkgindex = {} + pkglowlink = {} + cycles = [] + for pkg in packages: + tarjan(pkg) + +def tarjan(pkg): + global index,S,pkgindex,pkglowlink,cycles + pkgindex[pkg] = index + pkglowlink[pkg] = index + index += 1 + checked_deps.append(pkg) + S.append(pkg) + deps = [] + if pkg in pkgdeps: + deps = pkgdeps[pkg] + for dep in deps: + if dep not in pkgindex: + tarjan(dep) + pkglowlink[pkg] = min(pkglowlink[pkg],pkglowlink[dep]) + elif dep in S: + pkglowlink[pkg] = min(pkglowlink[pkg],pkgindex[dep]) + if pkglowlink[pkg] == pkgindex[pkg]: + dep = S.pop() + if pkg == dep: + return + path = pkg.name + while pkg != dep: + path = dep.repo + "/" + dep.name + ">" + path + dep = S.pop() + path = dep.name + ">" + path + if pkg.repo in repos: + circular_deps.append(path) + +def print_heading(heading): + print "" + print "=" * (len(heading) + 4) + print "= " + heading + " =" + print "=" * (len(heading) + 4) + +def print_subheading(subheading): + print "" + print subheading + print "-" * (len(subheading) + 2) + +def print_missdeps(pkgname,missdeps) : + for d in missdeps: + print pkgname + " : " + d + +def print_result(list, subheading): + if len(list) > 0: + list.sort() + print_subheading(subheading) + for item in list: + print item + +def print_results(): + print_result(missing_pkgbuilds, "Missing PKGBUILDs") + print_result(invalid_pkgbuilds, "Invalid PKGBUILDs") + print_result(mismatches, "Mismatched Pkgnames") + print_result(dups, "Duplicate PKGBUILDs") + print_result(invalid_archs, "Invalid Archs") + print_result(missing_deps, "Missing Dependencies") + print_result(missing_makedeps, "Missing Makedepends") + print_result(dep_hierarchy, "Repo Hierarchy for Dependencies") + print_result(makedep_hierarchy, "Repo Hierarchy for Makedepends") + print_result(circular_deps, "Circular Dependencies") + print_result(dbonly, "Packages found in db, but not in tree") + print_result(absonly,"Packages found in tree, but not in db") + print_subheading("Summary") + print "Missing PKGBUILDs: ", len(missing_pkgbuilds) + print "Invalid PKGBUILDs: ", len(invalid_pkgbuilds) + print "Mismatching PKGBUILD names: ", len(mismatches) + print "Duplicate PKGBUILDs: ", len(dups) + print "Invalid archs: ", len(invalid_archs) + print "Missing (make)dependencies: ", len(missing_deps)+len(missing_makedeps) + print "Repo hierarchy problems: ", len(dep_hierarchy)+len(makedep_hierarchy) + print "Circular dependencies: ", len(circular_deps) + print "In db, but not in tree: ", len(dbonly) + print "In tree, but not in db: ", len(absonly) + print "" + +def print_usage(): + print "" + print "Usage: ./check_packages.py [OPTION]" + print "" + print "Options:" + print " --abs-tree=<path[,path]> Check the specified tree(s) (default : /var/abs)" + print " --repos=<r1,r2,...> Check the specified repos (default : core,extra)" + print " --arch=<i686|x86_64> Check the specified arch (default : i686)" + print " --repo-dir=<path> Check the dbs at the specified path (default : /srv/ftp)" + print " -h, --help Show this help and exit" + print "" + print "Examples:" + print "\n Check core and extra in existing abs tree:" + print " ./check_packages.py --abs-tree=/var/abs --repos=core,extra --arch=i686" + print "\n Check community:" + print " ./check_packages.py --abs-tree=/var/abs --repos=community --arch=i686" + print "" + +## Default path to the abs root directory +absroots = ["/var/abs"] +## Default list of repos to check +repos = ['core', 'extra'] +## Default arch +arch = "i686" +## Default repodir +repodir = "/srv/ftp" + +try: + opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos=", + "arch=", "repo-dir="]) +except getopt.GetoptError: + print_usage() + sys.exit() +if opts != []: + for o, a in opts: + if o in ("--abs-tree"): + absroots = a.split(',') + elif o in ("--repos"): + repos = a.split(",") + elif o in ("--arch"): + arch = a + elif o in ("--repo-dir"): + repodir = a + else: + print_usage() + sys.exit() + if args != []: + print_usage() + sys.exit() + +for absroot in absroots: + if not os.path.isdir(absroot): + print "Error : the abs tree " + absroot + " does not exist" + sys.exit() + for repo in repos: + repopath = absroot + "/" + repo + if not os.path.isdir(repopath): + print("Warning : the repository " + repo + " does not exist in " + absroot) + +if not os.path.isdir(repodir): + print "Error: the repository directory %s does not exist" % repodir + sys.exit() +for repo in repos: + path = os.path.join(repodir,repo,'os',arch,repo + DBEXT) + if not os.path.isfile(path): + print "Error : repo DB %s : File not found" % path + sys.exit() + if not tarfile.is_tarfile(path): + print "Error : Cant open repo DB %s, not a valid tar file" % path + sys.exit() +# repos which need to be loaded +loadrepos = set([]) +for repo in repos: + loadrepos = loadrepos | set(get_repo_hierarchy(repo)) + +print_heading("Integrity Check " + arch + " of " + ",".join(repos)) +print("\nPerforming integrity checks...") + +print("==> parsing pkgbuilds") +parse_pkgbuilds(loadrepos,arch) + +# fill provisions +for name,pkg in packages.iteritems(): + for prov in pkg.provides: + provname=prov.split("=")[0] + if provname not in provisions: + provisions[provname] = [] + provisions[provname].append(pkg) + +# fill repopkgs +for name,pkg in packages.iteritems(): + if pkg.repo in repos: + repopkgs[name] = pkg + +print("==> parsing db files") +dbpkgs = parse_dbs(repos,arch) + +print("==> checking mismatches") +for name,pkg in repopkgs.iteritems(): + pkgdirname = pkg.path.split("/")[-1] + if name != pkgdirname and pkg.base != pkgdirname: + mismatches.append(name + " vs. " + pkg.path) + +print("==> checking archs") +for name,pkg in repopkgs.iteritems(): + archs = verify_archs(name,pkg.repo,pkg.archs) + invalid_archs.extend(archs) + +deph,makedeph = [],[] + +print("==> checking dependencies") +for name,pkg in repopkgs.iteritems(): + (deps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.deps) + pkgdeps[pkg] = deps + missing_deps.extend(missdeps) + deph.extend(hierarchy) + +print("==> checking makedepends") +for name,pkg in repopkgs.iteritems(): + (makedeps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.makedeps) + makepkgdeps[pkg] = makedeps + missing_makedeps.extend(missdeps) + makedeph.extend(hierarchy) + +print("==> checking hierarchy") +dep_hierarchy = check_hierarchy(deph) +makedep_hierarchy = check_hierarchy(makedeph) + +print("==> checking for circular dependencies") +# make sure pkgdeps is filled for every package +for name,pkg in packages.iteritems(): + if pkg not in pkgdeps: + (deps,missdeps,_) = verify_deps(name,pkg.repo,pkg.deps) + pkgdeps[pkg] = deps +find_scc(repopkgs.values()) + +print("==> checking for differences between db files and pkgbuilds") +for repo in repos: + for pkg in dbpkgs[repo]: + if not (pkg in repopkgs and repopkgs[pkg].repo == repo): + dbonly.append("%s/%s" % (repo,pkg)) +for name,pkg in repopkgs.iteritems(): + if not name in dbpkgs[pkg.repo]: + absonly.append("%s/%s" % (pkg.repo,name)) + +print_results() diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh new file mode 100755 index 0000000..5cd17e4 --- /dev/null +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -0,0 +1,149 @@ +#!/bin/bash + +# Usage : parse_pkgbuilds.sh arch <pkgbuilds_dir1,dir2,...> +# Example : parse_pkgbuilds.sh i686 /var/abs/core /var/abs/extra + +exit() { return; } + +splitpkg_overrides=('depends' 'optdepends' 'provides' 'conflicts') +variables=('pkgname' 'pkgbase' 'pkgver' 'pkgrel' 'makedepends' 'arch' ${splitpkg_overrides[@]}) +readonly -a variables splitpkg_overrides + +backup_package_variables() { + for var in ${splitpkg_overrides[@]}; do + indirect="${var}_backup" + eval "${indirect}=(\${$var[@]})" + done +} + +restore_package_variables() { + for var in ${splitpkg_overrides[@]}; do + indirect="${var}_backup" + if [ -n "${!indirect}" ]; then + eval "${var}=(\${$indirect[@]})" + else + unset ${var} + fi + done +} + +print_info() { + echo -e "%NAME%\n$pkgname\n" + echo -e "%VERSION%\n$pkgver-$pkgrel\n" + echo -e "%PATH%\n$dir\n" + + if [ -n "$pkgbase" ]; then + echo -e "%BASE%\n$pkgbase\n" + fi + + if [ -n "$arch" ]; then + echo "%ARCH%" + for i in ${arch[@]}; do echo $i; done + echo "" + fi + if [ -n "$depends" ]; then + echo "%DEPENDS%" + for i in ${depends[@]}; do + echo $i + done + echo "" + fi + if [ -n "$makedepends" ]; then + echo "%MAKEDEPENDS%" + for i in ${makedepends[@]}; do + echo $i + done + echo "" + fi + if [ -n "$conflicts" ]; then + echo "%CONFLICTS%" + for i in ${conflicts[@]}; do echo $i; done + echo "" + fi + if [ -n "$provides" ]; then + echo "%PROVIDES%" + for i in ${provides[@]}; do echo $i; done + echo "" + fi +} + +source_pkgbuild() { + ret=0 + dir=$1 + pkgbuild=$dir/PKGBUILD + for var in ${variables[@]}; do + unset ${var} + done + source $pkgbuild &>/dev/null || ret=$? + + # ensure $pkgname and $pkgver variables were found + if [ $ret -ne 0 -o -z "$pkgname" -o -z "$pkgver" ]; then + echo -e "%INVALID%\n$pkgbuild\n" + return 1 + fi + + if [ "${#pkgname[@]}" -gt "1" ]; then + pkgbase=${pkgbase:-${pkgname[0]}} + for pkg in ${pkgname[@]}; do + if [ "$(type -t package_${pkg})" != "function" ]; then + echo -e "%INVALID%\n$pkgbuild\n" + return 1 + else + backup_package_variables + pkgname=$pkg + while IFS= read -r line; do + var=${line%%=*} + var="${var#"${var%%[![:space:]]*}"}" # remove leading whitespace characters + for realvar in ${variables[@]}; do + if [ "$var" == "$realvar" ]; then + eval $line + break + fi + done + done < <(type package_${pkg}) + print_info + restore_package_variables + fi + done + else + echo + print_info + fi + + return 0 +} + +find_pkgbuilds() { + #Skip over some dirs + local d="${1##*/}" + if [ "$d" = "CVS" -o "$d" = ".svn" ]; then + return + fi + + if [ -f $1/PKGBUILD ]; then + source_pkgbuild $1 + return + fi + empty=1 + for dir in $1/*; do + if [ -d $dir ]; then + find_pkgbuilds $dir + unset empty + fi + done + if [ -n "$empty" ]; then + echo -e "%MISSING%\n$1\n" + fi +} + +if [ -z "$1" -o -z "$2" ]; then + exit 1 +fi + +CARCH=$1 +shift +for dir in "$@"; do + find_pkgbuilds $dir +done + +exit 0 diff --git a/cron-jobs/check_archlinux/setup.py b/cron-jobs/check_archlinux/setup.py new file mode 100644 index 0000000..b172752 --- /dev/null +++ b/cron-jobs/check_archlinux/setup.py @@ -0,0 +1,10 @@ +from distutils.core import setup, Extension + +alpm = Extension('alpm', + libraries = ['alpm'], + sources = ['alpm.c']) + +setup (name = 'Alpm', + version = '1.0', + description = 'Alpm bindings', + ext_modules = [alpm]) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists new file mode 100755 index 0000000..8bcfc0b --- /dev/null +++ b/cron-jobs/create-filelists @@ -0,0 +1,101 @@ +#!/bin/bash + +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" + +script_lock + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_lock ${repo} ${arch} || exit 1 + done +done + +#adjust the nice level to run at a lower priority +renice +10 -p $$ > /dev/null + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repodb="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" + filedb="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" + + if [ ! -f "${repodb}" ]; then + continue + fi + # get a list of package files defined in the repo db + mkdir -p "${WORKDIR}/db-dir-${repo}-${arch}" + bsdtar -xf "${repodb}" -C "${WORKDIR}/db-dir-${repo}-${arch}" + # This should actualy be faster than reading all the just extracted files + bsdtar -xOf "${repodb}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/db-${repo}-${arch}" + + # get a list of package files defined in the files db + mkdir -p "${WORKDIR}/files-current-dir-${repo}-${arch}" + if [ ! -f "${filedb}" ]; then + touch "${WORKDIR}/files-${repo}-${arch}" + else + bsdtar -xf "${filedb}" -C "${WORKDIR}/files-current-dir-${repo}-${arch}" + bsdtar -xOf "${filedb}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/files-${repo}-${arch}" + fi + done +done + +case "${FILESEXT}" in + *.gz) TAR_OPT="z" ;; + *.bz2) TAR_OPT="j" ;; + *.xz) TAR_OPT="J" ;; + *) die "Unknown compression type for FILESEXT=${FILESEXT}" ;; +esac + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + filedb="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" + + if [ ! -f "${WORKDIR}/db-${repo}-${arch}" ]; then + # remove any files db that might be in this empty repo + if [ -f "${filedb}" ]; then + rm -f "${filedb}" + fi + continue + fi + + # Check if updating the files db is needed + if ! diff -q "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}" >/dev/null; then + mkdir -p "${WORKDIR}/files-new-dir-${repo}-${arch}" + + # Include all unchanged file lists + # Note: deleted packages are implicitly excluded + for f in $(comm -12 "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}"); do + mv "${WORKDIR}/files-current-dir-${repo}-${arch}/${f%*-*${PKGEXT}}" \ + "${WORKDIR}/files-new-dir-${repo}-${arch}" + done + + # Create file lists for new packages + for f in $(comm -23 "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}"); do + tdir="${WORKDIR}/files-new-dir-${repo}-${arch}/${f%*-*${PKGEXT}}" + mkdir "${tdir}" + echo '%FILES%' > "${tdir}/files" + bsdtar --exclude=.* -tf "${FTP_BASE}/${repo}/os/${arch}/${f}" >> "${tdir}/files" + + # add desc and depends file from db + dbdir="${WORKDIR}/db-dir-${repo}-${arch}/${f%*-*${PKGEXT}}" + mv "${dbdir}/desc" "${tdir}/desc" + mv "${dbdir}/depends" "${tdir}/depends" + done + + # Create the actual file db + pushd "${WORKDIR}/files-new-dir-${repo}-${arch}" >/dev/null + bsdtar -c${TAR_OPT}f "${WORKDIR}/${arch}-${repo}${FILESEXT}" * + popd >/dev/null + mv -f "${WORKDIR}/${arch}-${repo}${FILESEXT}" "${filedb}" + ln -sf "${repo}${FILESEXT}" "${filedb%.tar.*}" + fi + done +done + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_unlock ${repo} ${arch} + done +done + +script_unlock diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer new file mode 100755 index 0000000..ca2e46b --- /dev/null +++ b/cron-jobs/devlist-mailer @@ -0,0 +1,28 @@ +#!/bin/bash +#Dummy helper to send email to arch-dev +# It does nothing if no output + +LIST="arch-dev-public@archlinux.org" +#LIST="aaronmgriffin@gmail.com" +FROM="repomaint@archlinux.org" + +SUBJECT="Repository Maintenance $(date +"%d-%m-%Y")" +if [ $# -ge 1 ]; then + SUBJECT="$1 $(date +"%d-%m-%Y")" +fi + +if [ $# -ge 2 ]; then + LIST="$2" +fi + +stdin="$(cat)" +#echo used to strip whitespace for checking for actual data +if [ -n "$(echo $stdin)" ]; then + +echo "Subject: $SUBJECT +To: $LIST +From: $FROM + +$stdin" | /usr/sbin/sendmail -F$FROM "$LIST" + +fi diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup new file mode 100755 index 0000000..09e8a49 --- /dev/null +++ b/cron-jobs/ftpdir-cleanup @@ -0,0 +1,113 @@ +#!/bin/bash + +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" + +clean_pkg() { + local pkg + local target + + if ! ${CLEANUP_DRYRUN}; then + for pkg in "$@"; do + if [ -h "$pkg" ]; then + rm -f "$pkg" + else + mv -f "$pkg" "$CLEANUP_DESTDIR" + touch "${CLEANUP_DESTDIR}/$(basename ${pkg})" + fi + done + fi +} + +script_lock + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_lock ${repo} ${arch} || exit 1 + done +done + +${CLEANUP_DRYRUN} && warning 'dry run mode is active' + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + continue + fi + # get a list of actual available package files + find "${FTP_BASE}/${repo}/os/${arch}" -xtype f -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/repo-${repo}-${arch}" + # get a list of package files defined in the repo db + bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/db-${repo}-${arch}" + + missing_pkgs=($(comm -13 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) + if [ ${#missing_pkgs[@]} -ge 1 ]; then + error "Missing packages in [${repo}] (${arch})..." + for missing_pkg in ${missing_pkgs[@]}; do + msg2 "${missing_pkg}" + done + fi + + old_pkgs=($(comm -23 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) + if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old packages from [${repo}] (${arch})..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + clean_pkg "${FTP_BASE}/${repo}/os/${arch}/${old_pkg}" + done + fi + done +done + +# get a list of all available packages in the pacakge pool +find "$FTP_BASE/${PKGPOOL}" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/pool" +# create a list of packages in our db +cat "${WORKDIR}/db-"* | sort -u > "${WORKDIR}/db" + +old_pkgs=($(comm -23 "${WORKDIR}/pool" "${WORKDIR}/db")) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old packages from package pool..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + clean_pkg "$FTP_BASE/${PKGPOOL}/${old_pkg}" + done +fi + +# cleanup of legacy $repo/os/any directories +for repo in ${PKGREPOS[@]}; do + if [ ! -d "${FTP_BASE}/${repo}/os/any" ]; then + continue + fi + if [ -n "$(find "${FTP_BASE}/${repo}/os/any" -type d -empty)" ]; then + msg "Removing empty legacy directory ${repo}/os/any" + ${CLEANUP_DRYRUN} || rmdir "${FTP_BASE}/${repo}/os/any" + continue + fi + find "${FTP_BASE}/${repo}/os/any" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/any-${repo}" + cat "${WORKDIR}/db-${repo}-"* | sort -u > "${WORKDIR}/all-${repo}" + + old_pkgs=($(comm -23 "${WORKDIR}/any-${repo}" "${WORKDIR}/all-${repo}")) + if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old packages from [${repo}] (any)..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + clean_pkg "${FTP_BASE}/${repo}/os/any/${old_pkg}" + done + fi +done + +old_pkgs=($(find ${CLEANUP_DESTDIR} -type f -name "*${PKGEXT}" -mtime +${CLEANUP_KEEP} -printf '%f\n')) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old packages from the cleanup directory..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + ${CLEANUP_DRYRUN} || rm -f "${CLEANUP_DESTDIR}/${old_pkg}" + done +fi + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_unlock ${repo} ${arch} + done +done + +script_unlock diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check new file mode 100755 index 0000000..d4f9694 --- /dev/null +++ b/cron-jobs/integrity-check @@ -0,0 +1,32 @@ +#!/bin/bash + +dirname="$(dirname $0)" + +. "${dirname}/../db-functions" +. "${dirname}/../config" + +script_lock + +if [ $# -ne 1 ]; then + die "usage: $(basename $0) <mailto>" +fi +mailto=$1 + +check() { + ${dirname}/check_archlinux/check_packages.py \ + --repos="${repos}" \ + --abs-tree="/srv/abs/rsync/${arch},/srv/abs/rsync/any" \ + --repo-dir="${FTP_BASE}" \ + --arch="${arch}" \ + 2>&1 | ${dirname}/devlist-mailer "Integrity Check ${arch}: ${repos}" "${mailto}" +} + +repos='core,extra,community' +arch='i686' +check + +repos='core,extra,community,multilib' +arch='x86_64' +check + +script_unlock diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs new file mode 100755 index 0000000..ee074bd --- /dev/null +++ b/cron-jobs/sourceballs @@ -0,0 +1,150 @@ +#!/bin/bash + +dirname="$(dirname $(readlink -e $0))" +. "${dirname}/../db-functions" +. "${dirname}/../config" +pushd "${WORKDIR}" >/dev/null + +script_lock + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_lock ${repo} ${arch} || exit 1 + done +done + +#adjust the nice level to run at a lower priority +renice +10 -p $$ > /dev/null + +# Create a readable file for each repo with the following format +# <pkgbase|pkgname> <pkgver>-<pkgrel> <arch> <license>[ <license>] +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + # Repo does not exist; skip it + if [ ! -f "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + continue + fi + bsdtar -xOf "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ + | awk '/^%NAME%/ { getline b }; + /^%BASE%/ { getline b }; + /^%VERSION%/ { getline v }; + /^%LICENSE%/,/^$/ { + if ( !/^%LICENSE%/ ) { l=l" "$0 } + }; + /^%ARCH%/ { + getline a; + printf "%s %s %s %s\n", b, v, a, l; + l=""; + }' + done | sort -u > "${WORKDIR}/db-${repo}" +done + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_unlock ${repo} ${arch} + done +done + +# Create a list of all available source package file names +find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" + +# Check for all packages if we need to build a source package +for repo in ${PKGREPOS[@]}; do + newpkgs=() + failedpkgs=() + while read line; do + pkginfo=(${line}) + pkgbase=${pkginfo[0]} + pkgver=${pkginfo[1]} + pkgarch=${pkginfo[2]} + pkglicense=(${pkginfo[@]:3}) + + # Should this packages be skipped? + if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then + continue + fi + # Commenting out, we'll sourceball everything + # Check if the license or .force file does not enforce creating a source package +# if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then +# continue +# fi + # Store the expected file name of the source package + echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" + + # Build the source package if its not already there + if ! grep -Fqx "${pkgbase}-${pkgver}${SRCEXT}" "${WORKDIR}/available-src-pkgs"; then + # Check if we had failed before + if in_array "${pkgbase}-${pkgver}${SRCEXT}" ${failedpkgs[@]}; then + continue + fi + + # Get the sources from svn + mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" + #svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ + # "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 + + # If it's on official repos, nor [libre], nor [libre-testing] + cp -r "${SVNREPO}/$repo/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ + cp -r "${SVNREPO}/libre/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ + cp -r "${SVNREPO}/libre-testing/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 + if [ $? -ge 1 ]; then + failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" + continue + fi + + # Build the actual source package + pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null + makepkg --nocolor --allsource --ignorearch # >/dev/null 2>&1 + if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then + mv "${pkgbase}-${pkgver}${SRCEXT}" "${ARCH_BASE}/${SRCPOOL}" + # Avoid creating the same source package for every arch + echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" + newpkgs[${#newpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" + else + failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" + fi + popd >/dev/null + fi + done < "${WORKDIR}/db-${repo}" + + if [ ${#newpkgs[@]} -ge 1 ]; then + msg "Adding source packages for [${repo}]..." + for new_pkg in ${newpkgs[@]}; do + msg2 "${new_pkg}" + done + fi + if [ ${#failedpkgs[@]} -ge 1 ]; then + msg "Failed to create source packages for [${repo}]..." + for failed_pkg in ${failedpkgs[@]}; do + msg2 "${failed_pkg}" + done + fi +done + +# Cleanup old source packages +cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" +cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort" +old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) + +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages..." + ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + if ! ${SOURCE_CLEANUP_DRYRUN}; then + mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + fi + done +fi + +old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -name "*${SRCEXT}" -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages from the cleanup directory..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + ${SOURCE_CLEANUP_DRYRUN} || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + done +fi + +script_unlock diff --git a/cron-jobs/sourceballs.force b/cron-jobs/sourceballs.force new file mode 100644 index 0000000..badf15d --- /dev/null +++ b/cron-jobs/sourceballs.force @@ -0,0 +1,4 @@ +faad2 +wxgtk +wxpython +glhack diff --git a/cron-jobs/sourceballs.skip b/cron-jobs/sourceballs.skip new file mode 100644 index 0000000..14d6f4b --- /dev/null +++ b/cron-jobs/sourceballs.skip @@ -0,0 +1,29 @@ +nexuiz-data +torcs-data +tremulous-data +ufoai-data +frogatto-data +vdrift-data +naev-data +btanks-data +wesnoth-data +texlive-bin +texlive-bibtexextra +texlive-core +texlive-fontsextra +texlive-formatsextra +texlive-games +texlive-genericextra +texlive-htmlxml +texlive-humanities +texlive-langcjk +texlive-langcyrillic +texlive-langextra +texlive-langgreek +texlive-latexextra +texlive-music +texlive-pictures +texlive-plainextra +texlive-pstricks +texlive-publishers +texlive-science diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 new file mode 100755 index 0000000..452208e --- /dev/null +++ b/cron-jobs/sourceballs2 @@ -0,0 +1,87 @@ +#!/bin/bash + +dirname="$(dirname $(readlink -e $0))" +. "${dirname}/../db-functions" +. "${dirname}/../config" +. "${MAKEPKGCONF}" + +pushd "${WORKDIR}" >/dev/null + +script_lock + +#adjust the nice level to run at a lower priority +renice +10 -p $$ > /dev/null + +# Create a list of all available source package file names +find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" + +# Steps +# Traverse the ABSLibre +# Makepkg --allsource every package +# Remove the old packages +pushd "${SVNREPO}" >/dev/null + +failedpkgs=() +for repo in ${PKGREPOS[@]}; do + pushd $repo >/dev/null + find . -maxdepth 1 -type d | while read pkg; do + pushd "${SVNREPO}/$repo/$pkg" >/dev/null + + [[ ! -e PKGBUILD ]] && { + warning "$repo/$pkg is not a package" + continue + } + + unset pkgbase pkgname + source PKGBUILD + pkgbase=${pkgbase:-$pkgname} + + echo "${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" + + # Skip already sourceballed + [[ -e "${SRCPKGDEST}/${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" ]] && \ + continue + + makepkg --allsource --ignorearch -c >/dev/null 2>&1 + + [[ $? -ne 0 ]] && \ + failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" + + done + popd >/dev/null +done + +# Cleanup old source packages +cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" +cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort" +old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) + +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages..." + ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + if ! ${SOURCE_CLEANUP_DRYRUN}; then + mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + fi + done +fi + +old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -name "*${SRCEXT}" -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages from the cleanup directory..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + ${SOURCE_CLEANUP_DRYRUN} || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + done +fi + +msg "Failed" +for _fail in ${failedpkgs[@]}; do + msg2 "$_fail" +done + + +script_unlock + diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db new file mode 100755 index 0000000..6ced4c1 --- /dev/null +++ b/cron-jobs/update-web-db @@ -0,0 +1,78 @@ +#!/bin/bash + +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" + +# setup paths +SPATH="/srv/http/archweb" +ENVPATH="/srv/http/archweb-env/bin/activate" + +# having "more important repos" last should make [core] trickle to the top of +# the updates list each hour rather than being overwhelmed by big [extra] and +# [community] updates +REPOS=('community-testing' 'multilib-testing' 'multilib' 'community' 'extra' 'testing' 'core') +LOGOUT="/tmp/archweb_update.log" + +# figure out what operation to perform +cmd="$(basename $0)" +if [[ $cmd != "update-web-db" && $cmd != "update-web-files-db" ]]; then + die "Invalid command name '$cmd' specified!" +fi + +script_lock + +# run at nice 5. it can churn quite a bit of cpu after all. +renice +5 -p $$ > /dev/null + +echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" + +# source our virtualenv if it exists +if [ -f "$ENVPATH" ]; then + . "$ENVPATH" +fi + +case "$cmd" in + update-web-db) + dbfileext="${DBEXT}" + flags="" + ;; + update-web-files-db) + dbfileext="${FILESEXT}" + flags="--filesonly" + ;; +esac + +# Lock the repos and get a copy of the db files to work on +for repo in ${REPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_lock ${repo} ${arch} || exit 1 + dbfile="/srv/ftp/${repo}/os/${arch}/${repo}${dbfileext}" + if [ -f "${dbfile}" ]; then + mkdir -p "${WORKDIR}/${repo}/${arch}" + cp "${dbfile}" "${WORKDIR}/${repo}/${arch}/${repo}${dbfileext}" + fi + repo_unlock ${repo} ${arch} + done +done + +# Run reporead on our db copy +pushd $SPATH >/dev/null +for repo in ${REPOS[@]}; do + for arch in ${ARCHES[@]}; do + dbcopy="${WORKDIR}/${repo}/${arch}/${repo}${dbfileext}" + if [ -f "${dbcopy}" ]; then + echo "Updating ${repo}-${arch}" >> "${LOGOUT}" + ./manage.py reporead ${flags} ${arch} "${dbcopy}" >> "${LOGOUT}" 2>&1 + echo "" >> "${LOGOUT}" + fi + done +done +popd >/dev/null +echo "" >> "${LOGOUT}" + +# rotate the file if it is getting big (> 10M), overwriting any old backup +if [[ $(stat -c%s "${LOGOUT}") -gt 10485760 ]]; then + mv "${LOGOUT}" "${LOGOUT}.old" +fi + +script_unlock diff --git a/cron-jobs/update-web-files-db b/cron-jobs/update-web-files-db new file mode 120000 index 0000000..0c2c4fa --- /dev/null +++ b/cron-jobs/update-web-files-db @@ -0,0 +1 @@ +update-web-db
\ No newline at end of file |