Merge repo-maintainer into dbscripts. Repo-maintainer will be abandoned as

of this commit.

Merge ssh://vparabola/srv/git/projects/repo-maintainer

Conflicts:
	db-functions

6 files changed, 44 insertions, 28 deletions

diff --git a/.gitignore b/.gitignore
index fb6241a..45688ab 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,5 @@
 local_config
 /config.local
 test/packages/*/*.pkg.tar.?z
+#*#
+.#*
\ No newline at end of file
diff --git a/clean_repo.py b/clean_repo.py
index e1a17c2..bc401b5 100755
--- a/clean_repo.py
+++ b/clean_repo.py
@@ -48,10 +48,13 @@ def remove_from_blacklist(path_to_db, blacklisted_names):
 def cleanup_nonfree_in_dir(directory, blacklisted_names):
     if "~" in directory:
         directory=(os.path.expanduser(directory))
+    pkglist=list()
     pkgs=pkginfo_from_files_in_dir(directory)
     for package in pkgs:
         if package["name"] in blacklisted_names:
             os.remove(package["location"])
+            pkglist.append(package)
+    return pkglist
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
@@ -64,10 +67,10 @@ if __name__ == "__main__":
 
     group_dir=parser.add_argument_group("Clean non-free packages in dir")
     group_dir.add_argument("-d", "--directory", type=str,
-                        help="directory to clean",)
+                        help="directory to clean")
 
     group_db=parser.add_argument_group("Clean non-free packages in db",
-                                       "All arguments need to be specified")
+                                       "All arguments need to be specified for db cleaning")
     group_db.add_argument("-b", "--database", type=str,
                           help="dabatase to clean")
     group_db.add_argument("-p", "--pending-file", type=str,
@@ -82,8 +85,8 @@ if __name__ == "__main__":
     elif not args.pending_file or not args.whitelist_file \
             and args.database:
         parser.print_help()
-    else:
-        blacklisted=listado(args.blacklist_file)
+
+    blacklisted=listado(args.blacklist_file)
 
     if args.database:
         whitelisted=listado(args.whitelist_file)
@@ -94,3 +97,4 @@ if __name__ == "__main__":
 
     if args.directory:
         cleanup_nonfree_in_dir(args.directory, blacklisted)
+    
diff --git a/db-functions b/db-functions
index ff38d62..20bfa0e 100644
--- a/db-functions
+++ b/db-functions
@@ -497,7 +497,7 @@ arch_repo_add() {
 
 	# package files might be relative to repo dir
 	pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null
-	repo-add -q "${repo}${DBEXT}" ${pkgs[@]} \
+	repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \
 		|| error "repo-add ${repo}${DBEXT} ${pkgs[@]}"
 	/usr/bin/repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \
 		|| error "repo-add -f ${repo}${FILESEXT} ${pkgs[@]}"
@@ -516,7 +516,7 @@ arch_repo_remove() {
 		error "No database found at '${dbfile}'"
 		return 1
 	fi
-	repo-remove -q "${dbfile}" ${pkgs[@]} \
+	repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \
 		|| error "repo-remove ${dbfile} ${pkgs[@]}"
 	/usr/bin/repo-remove -q "${filesfile}" ${pkgs[@]} \
 		|| error "repo-remove ${filesfile} ${pkgs[@]}"
diff --git a/filter.py b/filter.py
index c71f54d..5d90bdd 100755
--- a/filter.py
+++ b/filter.py
@@ -1,4 +1,4 @@
- #! /usr/bin/python
+#! /usr/bin/python
 #-*- encoding: utf-8 -*-
 from glob import glob
 from repm.config import *
@@ -188,11 +188,17 @@ if __name__ == "__main__":
     parser.add_argument("-k", "--blacklist-file", type=str,
                         help="File containing blacklisted names",
                         required=True,)
-    parser.add_argument("-c", "--rsync-command", type=str,
-                        help="This command will be run to get a pkg list",
+    parser.add_argument("-f", "--rsout-file", type=str,
+                        help="This file will be read to get a pkg list",
                         required=True,)
     args=parser.parse_args()
-    rsout=check_output(args.rsync_command.split())
+    try:
+        fsock=open(args.rsout_file, "r")
+        rsout=fsock.read()
+    except IOError:
+        print("%s is not readable" % args.rsout_file)
+    finally:
+        fsock.close()
     packages=pkginfo_from_rsync_output(rsout)
-    rsyncBlaclist_from_blacklist(packages, listado(args.blacklist_file),
+    rsyncBlacklist_from_blacklist(packages, listado(args.blacklist_file),
                                  args.rsync_exclude_file)
diff --git a/local_config.example b/local_config.example
index 0a827f4..05694e9 100644
--- a/local_config.example
+++ b/local_config.example
@@ -1,6 +1,6 @@
 # Mirror options
 mirror="mirrors.eu.kernel.org"
-mirrorpath="::mirrors/archlinux"
+mirrorpath="mirrors/archlinux"
 
 # Directories: they should end without /
 paraboladir=~/parabolagnulinux.org
@@ -13,11 +13,14 @@ licenses_dir=${docs_dir}/pending_licenses
 
 # Files
 logname=${paraboladir}/log/$(date -u +%Y%m%d-%H:%M)-repo-maintainer.log
+rsout_file=${tempdir}/rsout
+rsync_not_needed=${tempdir}/rsync_not_needed
+
+rsync_blacklist=${docs_dir}/rsyncBlacklist
+
 blacklist=${docs_dir}/blacklist.txt
 whitelist=${docs_dir}/whitelist.txt
-rsync_blacklist=${docs_dir}/rsyncBlacklist
-rsync_not_needed=${tmp}/rsync_not_needed
 
 # Rsync commands
-rsync_list_command="rsync\ -ptgoL\ --list-only\ "
-rsync_update_command="rsync -ptgoL --exclude='*.abs.tar.*'"
+rsync_list_command="rsync -rptgoL --exclude='*.abs.tar.*' --list-only --no-motd "
+rsync_update_command="rsync -rptgoL --exclude='*.abs.tar.*' --no-motd "
diff --git a/repo-update b/repo-update
index 59a0658..2354e0a 100755
--- a/repo-update
+++ b/repo-update
@@ -1,24 +1,25 @@
 #!/bin/bash
 # -*- coding: utf-8 -*-
 
-source ./config
-source ./local_config
-source ./libremessages
+source config
+source local_config
+source libremessages
 
 for repo in ${PKGREPOS[@]}; do
     for arch in ${ARCHES[@]} 'any'; do
 	msg "Syncing ${repo} ${arch}"
-	filter.py -r "${rsync_blacklist}" -k "${blacklist}" -c \
-	    \"${rsync_list_command}\ \
-	    ${mirror}${mirrorpath}/${repo}/os/${arch}\ \
-	    ${repodir}/${repo}/\"
-	find ${repodir}/${repo} -name *${PKGEXT} -print \
-	    > ${rsync_not_needed}
+	${rsync_list_command} \
+	    ${mirror}::${mirrorpath}/${repo}/os/${arch}/ \
+	    ${repodir}/${repo}/ > ${rsout_file}
+	filter.py -r ${rsync_blacklist} -k ${blacklist} \
+	    -f ${rsout_file}
+	find ${repodir}/${repo} -name *${PKGEXT} \
+	    -fprint ${rsync_not_needed}
 	${rsync_update_command} \
-	    ${mirror}${mirrorpath}/${repo}/os/${arch} \
-	    ${repodir}/${repo} \
 	    --exclude-from=${rsync_blacklist} \
 	    --exclude-from=${rsync_not_needed}
+	    ${mirror}::${mirrorpath}/${repo}/os/${arch}/ \
+	    ${repodir}/${repo}/
     done
     for arch in ${ARCHES[@]}; do
 	if [ -r ${repodir}/${repo}/os/${arch}/${repo}${DBEXT} ]; then
@@ -26,7 +27,7 @@ for repo in ${PKGREPOS[@]}; do
 		-p ${docs_dir}/pending-${repo} \
 		-b ${repodir}/${repo}/${repo}${DBEXT}
 	fi
-	clean_repo.py -k ${blacklist} -d ${repodir}/${repo}
+	python clean_repo.py -k ${blacklist} -d ${repodir}/${repo}
     done
 done