From 49a0260b06f9d24c790631cb4e2a4f63fce3a8c3 Mon Sep 17 00:00:00 2001 From: Parabola Date: Fri, 17 Sep 2010 09:55:19 -0700 Subject: Upload version --- TODO | 0 __init__.py | 0 changelog | 2 + get_license.sh | 55 ++++++++++++++ pato2.py | 232 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ repo-state | 11 +++ 6 files changed, 300 insertions(+) create mode 100644 TODO create mode 100644 __init__.py create mode 100644 changelog create mode 100755 get_license.sh create mode 100644 pato2.py create mode 100755 repo-state diff --git a/TODO b/TODO new file mode 100644 index 0000000..e69de29 diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/changelog b/changelog new file mode 100644 index 0000000..0e89f2f --- /dev/null +++ b/changelog @@ -0,0 +1,2 @@ +repo-maintainer 1.0 + * Known working version diff --git a/get_license.sh b/get_license.sh new file mode 100755 index 0000000..a7241a1 --- /dev/null +++ b/get_license.sh @@ -0,0 +1,55 @@ +#!/bin/sh +# -*- coding: utf-8 -*- + + # get_license.sh + # Copyright 2010 Joshua Ismael Haase Hernández + + # ---------- GNU General Public License 3 ---------- + + # This file is part of Parabola. + + # Parabola is free software: you can redistribute it and/or modify + # it under the terms of the GNU General Public License as published by + # the Free Software Foundation, either version 3 of the License, or + # (at your option) any later version. + + # Parabola is distributed in the hope that it will be useful, + # but WITHOUT ANY WARRANTY; without even the implied warranty of + # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + # GNU General Public License for more details. + + # You should have received a copy of the GNU General Public License + # along with Parabola. If not, see . + +docs="/home/parabolavnx/parabolagnulinux.org/docs" +repo="/home/parabolavnx/parabolagnulinux.org/repo" +dir="$docs/pending-licenses" + +echo "Cleaning $dir" +rm -rf $dir/* + +tempdir=$(mktemp -d) +cd $tempdir + +a=($(cut -d: -f1 $docs/pending*.txt)) +echo ${a[@]} + +for x in ${a[@]}; do + b=( $(ls $repo/*/os/*/$x*) ) + for y in ${b[@]}; do + echo "chmod +r $y" + chmod +r $y + echo "tar -xf $y usr/share/licenses" + bsdtar -xf $y usr/share/licenses + echo "chmod -r $y" + chmod -r $y + done +done + +mv usr/share/licenses/* $dir + +cd + +rm -rf $tempdir + +exit 0 \ No newline at end of file diff --git a/pato2.py b/pato2.py new file mode 100644 index 0000000..6d8a981 --- /dev/null +++ b/pato2.py @@ -0,0 +1,232 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +""" + parabola.py + Copyright 2009 Rafik Mas'ad + Copyright 2010 Joshua Ismael Haase Hernández + + ---------- GNU General Public License 3 ---------- + + This file is part of Parabola. + + Parabola is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Parabola is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Parabola. If not, see . + + +""" + +import tarfile, commands +from glob import glob +from user import home +from os.path import isdir, isfile, realpath +# ---------- Config Variables Start Here ---------- # + +time__ = commands.getoutput("date +%Y%m%d-%H:%M") + +# Mirror Parameters +mirror = "mirrors.kernel.org" +mirrorpath = "::mirrors/archlinux" + +# Directories and files +## Optionals +path = home + "/parabolagnulinux.org" +docs = path + "/docs" +logdir = path + "/log" +## Must be defined +logname= logdir + "/" + time__ + "-repo-maintainer.log" +repodir= path + "/repo" +tmp = home + "/tmp" +archdb = tmp + "/db" +emptydb= path + "/files/repo-empty.db.tar.gz" + +# Repo, arch, and other folders to use for repo +repo_list = ("core", "extra", "community","multilib") +dir_list = ("pool",) +arch_list = ("i686", "x86_64") +other = ("any",) + +# Output +output = True +verbose = False + +# Files +blacklist = docs + "/blacklist.txt" +whitelist = docs + "/whitelist.txt" +pending = docs + "/pending" +rsyncBlacklist = docs + "/rsyncBlacklist" + +# ---------- Config Variables End Here---------- # + +def printf(text,output_=output): + """Guarda el texto en la variable log y puede imprimir en pantalla.""" + log_file = open(logname, 'a') + log_file.write("\n" + str(text) + "\n") + log_file.close() + if output_: print (str(text) + "\n") + +def listado(filename_): + """Obtiene una lista de paquetes de un archivo.""" + archivo = open(filename_,"r") + lista = archivo.read().split("\n") + archivo.close() + return [pkg.split(":")[0] for pkg in lista if pkg] + +def db(repo_,arch_): + """Construye un nombre para sincronizar una base de datos.""" + return "/%s/os/%s/%s.db.tar.gz" % (repo_, arch_, repo_) + +def packages(repo_, arch_, expr="*"): + """ Get packages on a repo, arch folder """ + return tuple( glob( repodir + "/" + repo_ + "/os/" + arch_ + "/" + expr ) ) + +def sync_all_repo(verbose_=verbose): + folders = ",".join(repo_list + dir_list) + cmd_ = "rsync -av --delete-after --delay-updates " + mirror + mirrorpath + "/{" + folders + "} " + repodir + printf(cmd_) + a=commands.getoutput(cmd_) + if verbose_: printf(a) + +def get_from_desc(desc, var,db_tar_file=False): + """ Get a var from desc file """ + desc = desc.split("\n") + return desc[desc.index(var)+1] + +def get_info(repo_,arch_,db_tar_file=False,verbose_=verbose): + """ Makes a list of package name, file and license """ + info=list() + # Extract DB tar.gz + commands.getoutput("mkdir -p " + archdb) + if not db_tar_file: + db_tar_file = repodir + db(repo_,arch_) + if isfile(db_tar_file): + db_open_tar = tarfile.open(db_tar_file, 'r:gz') + else: + printf("No db_file %s" % db_tar_file) + return(tuple()) + for file in db_open_tar.getmembers(): + db_open_tar.extract(file, archdb) + db_open_tar.close() + # Get info from file + for dir_ in glob(archdb + "/*"): + if isdir(dir_) and isfile(dir_ + "/desc"): + pkg_desc_file = open(dir_ + "/desc", "r") + desc = pkg_desc_file.read() + pkg_desc_file.close() + info.append(( get_from_desc(desc,"%NAME%"), + dir_.split("/")[-1], + get_from_desc(desc,"%LICENSE%") )) + if verbose_: printf(info) + commands.getoutput("rm -r %s/*" % archdb) + return tuple(info) + +def make_pending(repo_,arch_,info_): + """ Si los paquetes no están en blacklist ni whitelist y la licencia contiene "custom" los agrega a pending""" + search = tuple( listado(blacklist) + listado (whitelist) ) + if verbose: printf("blaclist + whitelist= " + str(search) ) + lista_=list() + for (name,pkg_,license_) in info_: + if "custom" in license_: + if name not in search: + lista_.append( (name, license_ ) ) + elif not name: + printf( pkg_ + " package has no %NAME% attibute " ) + if verbose: printf( lista_ ) + a=open( pending + "-" + repo_ + ".txt", "w" ).write( + "\n".join([name + ":" + license_ for (name,license_) in lista_]) ) + +def remove_from_blacklist(repo_,arch_,info_,blacklist_): + """ Check the blacklist and remove packages on the db""" + lista_=list() + pack_=list() + for (name_, pkg_, license_) in info_: + if name_ in blacklist_: + lista_.append(name_) + for p in packages(repo_,arch_,pkg_ + "*"): + pack_.append(p) + if lista_: + lista_=" ".join(lista_) + com_ = "repo-remove " + repodir + db(repo_,arch_) + " " + lista_ + printf(com_) + a = commands.getoutput(com_) + if verbose: printf(a) + if pack_: + pack_=" ".join(pack_) + com_="chmod a-r " + pack_ + printf(com_) + a=commands.getoutput(com_) + if verbose: printf(a) + +def link(repo_,arch_,file_): + """ Makes a link in the repo for the package """ + cmd_="ln -sf " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ + a=commands.getoutput(cmd_) + if verbose: + printf(cmd_ + a) + +def add_free_repo(verbose_=verbose): + for repo_ in repo_list: + for arch_ in arch_list: + lista_=list() + for file_ in glob(repodir + "/free/" + repo_ + "/os/" + arch_ + "/*"): + lista_.append(file_) + link(repo_,arch_,file_) + for dir_ in other: + for file_ in glob(repodir + "/free/" + repo_ + "/os/" + dir_ + "/*"): + lista_.append(file_) + link(repo_,arch_,file_) + if lista_: + lista_=" ".join(lista_) + if verbose: printf(lista_) + cmd_="repo-add " + repodir + db(repo_,arch_) + " " + lista_ + printf(cmd_) + a=commands.getoutput(cmd_) + if verbose: printf(a) + +def get_licenses(verbose_=verbose): + """ Extract the license from packages in repo_,arch_ and in pending_ file""" + cmd_=home + "/usr/bin/get_license.sh" + printf(cmd_) + a=commands.getoutput(cmd_) + if verbose_: printf(a) + +if __name__ == "__main__": + from time import time + start_time = time() + def minute(): + return str(round((time() - start_time)/60, 1)) + + printf(" Cleaning %s folder " % (tmp) ) + commands.getoutput("rm -r %s/*" % tmp) + printf(" Syncing repo") + sync_all_repo(True) + + printf(" Updating databases and pending files lists: minute %s \n" % minute() ) + for repo in repo_list: + for arch in arch_list: + printf( "\n" + repo + "-" + arch + "\n" ) + printf( "Get info: minute %s " % minute() ) + info=get_info(repo,arch) + printf( "Make pending: minute %s" % minute() ) + make_pending(repo,arch,info) + printf( "Update DB: minute %s" % minute() ) + remove_from_blacklist( + repo, arch, info, tuple( listado(blacklist) + listado(pending + "-" + repo + ".txt") ) ) + + printf("Adding Parabola Packages: minute %s\n" % minute() ) + add_free_repo(True) + + printf("Extracting licenses in pending: minute %s" % minute() ) + get_licenses() + + printf("\n\nDelay: %s minutes \n" % minute()) + diff --git a/repo-state b/repo-state new file mode 100755 index 0000000..319db83 --- /dev/null +++ b/repo-state @@ -0,0 +1,11 @@ +#!/bin/sh + +a=$(date +%Y%m%d+%H%M) +mkdir -p $a +cd $a + +. /home/joshpar/programas/arch2parabola/repo-list-diff +# scp parabolavnx@parabolagnulinux.org:~/tmp/rsyncBlacklist.txt ./ + +cd .. +exit 0 \ No newline at end of file -- cgit v1.2.3-2-g168b From 847679d7a5e594efce5f8df73ca6751fab9ce591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 17 Sep 2010 12:34:02 -0500 Subject: Added TODO list --- TODO | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/TODO b/TODO index e69de29..11ef51a 100644 --- a/TODO +++ b/TODO @@ -0,0 +1,6 @@ +* Test Suite + + - Review all repo efectively + - Remove all blacklisted packages + - Get pending list right + - Extract licenses all right \ No newline at end of file -- cgit v1.2.3-2-g168b From c6787b4d56fc585aca5b5f5422aea0f2c85a5db1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 28 Sep 2010 21:21:05 -0500 Subject: Added sources to the repo syncing command --- .gitignore | 1 + pato2.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e4e5f6c --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*~ \ No newline at end of file diff --git a/pato2.py b/pato2.py index 6d8a981..df81b73 100644 --- a/pato2.py +++ b/pato2.py @@ -51,7 +51,7 @@ emptydb= path + "/files/repo-empty.db.tar.gz" # Repo, arch, and other folders to use for repo repo_list = ("core", "extra", "community","multilib") -dir_list = ("pool",) +dir_list = ("pool","sources") arch_list = ("i686", "x86_64") other = ("any",) -- cgit v1.2.3-2-g168b From 6c1729bf9d94407b36ac5717c254dacf8dffef3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 12 Oct 2010 11:51:43 -0500 Subject: If it can't open db_tar_file, print a warning and continue. --- pato2.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pato2.py b/pato2.py index df81b73..58e03f8 100644 --- a/pato2.py +++ b/pato2.py @@ -109,7 +109,11 @@ def get_info(repo_,arch_,db_tar_file=False,verbose_=verbose): if not db_tar_file: db_tar_file = repodir + db(repo_,arch_) if isfile(db_tar_file): - db_open_tar = tarfile.open(db_tar_file, 'r:gz') + try: + db_open_tar = tarfile.open(db_tar_file, 'r:gz') + except tarfile.ReadError: + printf("No valid db_file %s" % db_tar_file) + return(tuple()) else: printf("No db_file %s" % db_tar_file) return(tuple()) -- cgit v1.2.3-2-g168b From 473de5be358e3bc1c0c490622e9b2c47d2bfbdf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 12 Oct 2010 12:33:33 -0500 Subject: Deleted unused variables --- pato2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pato2.py b/pato2.py index 58e03f8..78d8c55 100644 --- a/pato2.py +++ b/pato2.py @@ -47,7 +47,6 @@ logname= logdir + "/" + time__ + "-repo-maintainer.log" repodir= path + "/repo" tmp = home + "/tmp" archdb = tmp + "/db" -emptydb= path + "/files/repo-empty.db.tar.gz" # Repo, arch, and other folders to use for repo repo_list = ("core", "extra", "community","multilib") -- cgit v1.2.3-2-g168b From b8619985289cde1591aa40c7df2c948fdbb8c30a Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 31 Jan 2011 09:15:56 -0800 Subject: changed mirror --- pato2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pato2.py b/pato2.py index 78d8c55..cf40809 100644 --- a/pato2.py +++ b/pato2.py @@ -34,7 +34,7 @@ from os.path import isdir, isfile, realpath time__ = commands.getoutput("date +%Y%m%d-%H:%M") # Mirror Parameters -mirror = "mirrors.kernel.org" +mirror = "mirrors.eu.kernel.org" mirrorpath = "::mirrors/archlinux" # Directories and files @@ -49,7 +49,7 @@ tmp = home + "/tmp" archdb = tmp + "/db" # Repo, arch, and other folders to use for repo -repo_list = ("core", "extra", "community","multilib") +repo_list = ("core", "extra", "community", "testing", "community-testing", "multilib") dir_list = ("pool","sources") arch_list = ("i686", "x86_64") other = ("any",) @@ -90,7 +90,7 @@ def packages(repo_, arch_, expr="*"): def sync_all_repo(verbose_=verbose): folders = ",".join(repo_list + dir_list) - cmd_ = "rsync -av --delete-after --delay-updates " + mirror + mirrorpath + "/{" + folders + "} " + repodir + cmd_ = "rsync -av --progress --delete-after --delay-updates " + mirror + mirrorpath + "/{" + folders + "} " + repodir printf(cmd_) a=commands.getoutput(cmd_) if verbose_: printf(a) -- cgit v1.2.3-2-g168b From a5872f3b259b97125089866169d06aa9a4d78e23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 7 Feb 2011 12:30:30 -0600 Subject: moved config out from pato2 --- config.py | 32 ++++++++++++++++++++++++++++++++ pato2.py | 37 +------------------------------------ 2 files changed, 33 insertions(+), 36 deletions(-) create mode 100644 config.py diff --git a/config.py b/config.py new file mode 100644 index 0000000..e2fc350 --- /dev/null +++ b/config.py @@ -0,0 +1,32 @@ +time__ = commands.getoutput("date +%Y%m%d-%H:%M") + +# Mirror Parameters +mirror = "mirrors.eu.kernel.org" +mirrorpath = "::mirrors/archlinux" + +# Directories and files +## Optionals +path = home + "/parabolagnulinux.org" +docs = path + "/docs" +logdir = path + "/log" +## Must be defined +logname= logdir + "/" + time__ + "-repo-maintainer.log" +repodir= path + "/repo" +tmp = home + "/tmp" +archdb = tmp + "/db" + +# Repo, arch, and other folders to use for repo +repo_list = ("core", "extra", "community", "testing", "community-testing", "multilib") +dir_list = ("pool","sources") +arch_list = ("i686", "x86_64") +other = ("any",) + +# Output +output = True +verbose = False + +# Files +blacklist = docs + "/blacklist.txt" +whitelist = docs + "/whitelist.txt" +pending = docs + "/pending" +rsyncBlacklist = docs + "/rsyncBlacklist" diff --git a/pato2.py b/pato2.py index cf40809..3ed57b2 100644 --- a/pato2.py +++ b/pato2.py @@ -24,47 +24,12 @@ """ +from repo_maintainer.config import * import tarfile, commands from glob import glob from user import home from os.path import isdir, isfile, realpath -# ---------- Config Variables Start Here ---------- # - -time__ = commands.getoutput("date +%Y%m%d-%H:%M") - -# Mirror Parameters -mirror = "mirrors.eu.kernel.org" -mirrorpath = "::mirrors/archlinux" - -# Directories and files -## Optionals -path = home + "/parabolagnulinux.org" -docs = path + "/docs" -logdir = path + "/log" -## Must be defined -logname= logdir + "/" + time__ + "-repo-maintainer.log" -repodir= path + "/repo" -tmp = home + "/tmp" -archdb = tmp + "/db" - -# Repo, arch, and other folders to use for repo -repo_list = ("core", "extra", "community", "testing", "community-testing", "multilib") -dir_list = ("pool","sources") -arch_list = ("i686", "x86_64") -other = ("any",) - -# Output -output = True -verbose = False - -# Files -blacklist = docs + "/blacklist.txt" -whitelist = docs + "/whitelist.txt" -pending = docs + "/pending" -rsyncBlacklist = docs + "/rsyncBlacklist" - -# ---------- Config Variables End Here---------- # def printf(text,output_=output): """Guarda el texto en la variable log y puede imprimir en pantalla.""" -- cgit v1.2.3-2-g168b From 44dc101393495d876a4fab40689c1d6c8b307c20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 7 Feb 2011 12:32:16 -0600 Subject: Added headers to config.py --- config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config.py b/config.py index e2fc350..df837d4 100644 --- a/config.py +++ b/config.py @@ -1,3 +1,6 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + time__ = commands.getoutput("date +%Y%m%d-%H:%M") # Mirror Parameters -- cgit v1.2.3-2-g168b From 937bf24bacdd29629b5efd2c949c0a6e4d17b163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 7 Feb 2011 17:38:33 -0600 Subject: Making hardlinks instead of symbolic --- config.py | 2 ++ pato2.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index df837d4..8a55cf7 100644 --- a/config.py +++ b/config.py @@ -8,10 +8,12 @@ mirror = "mirrors.eu.kernel.org" mirrorpath = "::mirrors/archlinux" # Directories and files + ## Optionals path = home + "/parabolagnulinux.org" docs = path + "/docs" logdir = path + "/log" + ## Must be defined logname= logdir + "/" + time__ + "-repo-maintainer.log" repodir= path + "/repo" diff --git a/pato2.py b/pato2.py index 3ed57b2..cd08ca0 100644 --- a/pato2.py +++ b/pato2.py @@ -24,7 +24,7 @@ """ -from repo_maintainer.config import * +from repm.config import * import tarfile, commands from glob import glob @@ -136,7 +136,7 @@ def remove_from_blacklist(repo_,arch_,info_,blacklist_): def link(repo_,arch_,file_): """ Makes a link in the repo for the package """ - cmd_="ln -sf " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ + cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ a=commands.getoutput(cmd_) if verbose: printf(cmd_ + a) -- cgit v1.2.3-2-g168b From 100970b35ed77a8f617a7071aba2618e038923ba Mon Sep 17 00:00:00 2001 From: Joshua Haase Date: Mon, 7 Feb 2011 16:51:06 -0800 Subject: Use free repo on new location --- pato2.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pato2.py b/pato2.py index cf40809..9cbd3cb 100644 --- a/pato2.py +++ b/pato2.py @@ -40,6 +40,7 @@ mirrorpath = "::mirrors/archlinux" # Directories and files ## Optionals path = home + "/parabolagnulinux.org" +free_path = path + "/free/" docs = path + "/docs" logdir = path + "/log" ## Must be defined @@ -56,7 +57,7 @@ other = ("any",) # Output output = True -verbose = False +verbose = True # Files blacklist = docs + "/blacklist.txt" @@ -180,11 +181,11 @@ def add_free_repo(verbose_=verbose): for repo_ in repo_list: for arch_ in arch_list: lista_=list() - for file_ in glob(repodir + "/free/" + repo_ + "/os/" + arch_ + "/*"): + for file_ in glob(free_path + repo_ + "/os/" + arch_ + "/*"): lista_.append(file_) link(repo_,arch_,file_) for dir_ in other: - for file_ in glob(repodir + "/free/" + repo_ + "/os/" + dir_ + "/*"): + for file_ in glob(free_path + repo_ + "/os/" + dir_ + "/*"): lista_.append(file_) link(repo_,arch_,file_) if lista_: -- cgit v1.2.3-2-g168b From 8c4813edd80c08e03c70ada2cb46a3597e493aaf Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 7 Feb 2011 16:54:47 -0800 Subject: 'Configuration out of pato' --- pato2.py | 40 ++-------------------------------------- 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/pato2.py b/pato2.py index 9cbd3cb..dd32403 100644 --- a/pato2.py +++ b/pato2.py @@ -24,48 +24,12 @@ """ +from repm.config import * import tarfile, commands from glob import glob from user import home from os.path import isdir, isfile, realpath -# ---------- Config Variables Start Here ---------- # - -time__ = commands.getoutput("date +%Y%m%d-%H:%M") - -# Mirror Parameters -mirror = "mirrors.eu.kernel.org" -mirrorpath = "::mirrors/archlinux" - -# Directories and files -## Optionals -path = home + "/parabolagnulinux.org" -free_path = path + "/free/" -docs = path + "/docs" -logdir = path + "/log" -## Must be defined -logname= logdir + "/" + time__ + "-repo-maintainer.log" -repodir= path + "/repo" -tmp = home + "/tmp" -archdb = tmp + "/db" - -# Repo, arch, and other folders to use for repo -repo_list = ("core", "extra", "community", "testing", "community-testing", "multilib") -dir_list = ("pool","sources") -arch_list = ("i686", "x86_64") -other = ("any",) - -# Output -output = True -verbose = True - -# Files -blacklist = docs + "/blacklist.txt" -whitelist = docs + "/whitelist.txt" -pending = docs + "/pending" -rsyncBlacklist = docs + "/rsyncBlacklist" - -# ---------- Config Variables End Here---------- # def printf(text,output_=output): """Guarda el texto en la variable log y puede imprimir en pantalla.""" @@ -172,7 +136,7 @@ def remove_from_blacklist(repo_,arch_,info_,blacklist_): def link(repo_,arch_,file_): """ Makes a link in the repo for the package """ - cmd_="ln -sf " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ + cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ a=commands.getoutput(cmd_) if verbose: printf(cmd_ + a) -- cgit v1.2.3-2-g168b From 317c0b412642b5268e8408f9d87e19261370cd32 Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 7 Feb 2011 18:25:37 -0800 Subject: fixed errors --- config.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 config.py diff --git a/config.py b/config.py new file mode 100644 index 0000000..9889c3f --- /dev/null +++ b/config.py @@ -0,0 +1,41 @@ + +#!/usr/bin/python +# -*- coding: utf-8 -*- +from user import home +import commands + +time__ = commands.getoutput("date +%Y%m%d-%H:%M") + +# Mirror Parameters +mirror = "mirrors.eu.kernel.org" +mirrorpath = "::mirrors/archlinux" + +# Directories and files + +## Optionals +path = home + "/parabolagnulinux.org" +docs = path + "/docs" +logdir = path + "/log" + +## Must be defined +logname= logdir + "/" + time__ + "-repo-maintainer.log" +repodir= path + "/repo" +tmp = home + "/tmp" +archdb = tmp + "/db" +free_path= path + "/free/" + +# Repo, arch, and other folders to use for repo +repo_list = ("core", "extra", "community", "testing", "community-testing", "multilib") +dir_list = ("pool","sources") +arch_list = ("i686", "x86_64") +other = ("any",) + +# Output +output = True +verbose = False + +# Files +blacklist = docs + "/blacklist.txt" +whitelist = docs + "/whitelist.txt" +pending = docs + "/pending" +rsyncBlacklist = docs + "/rsyncBlacklist" -- cgit v1.2.3-2-g168b From f837844242142599d1dee84309468624f6dd2048 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 00:31:56 -0600 Subject: Added classes and exceptions --- config.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/config.py b/config.py index 223257d..6dcd348 100644 --- a/config.py +++ b/config.py @@ -39,3 +39,27 @@ blacklist = docs + "/blacklist.txt" whitelist = docs + "/whitelist.txt" pending = docs + "/pending" rsyncBlacklist = docs + "/rsyncBlacklist" + +# Classes and Exceptions +class NonValidFile(ValueError): pass +class NonValidDir(ValueError): pass +class NonValidCommand(ValueError): pass + +class Package: + """ An object that has information about a package. """ + package_info={ "name" : False, + "version" : False, + "arch" : False, + "license" : False, + "location": False} + + def __setitem__(self, key, item): + return self.package_info.__setitem__(key, item) + + def __getitem__(self, key): + return self.package_info.__getitem__(key) + + def __unicode__(self): + return str(self.package_info) + + -- cgit v1.2.3-2-g168b From 039c18a9ab6259a13285a22d84390bba2c35642e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 00:37:29 -0600 Subject: Removed extra import commands: * commands and home were loaded on repm.config * realpath is not used --- pato2.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pato2.py b/pato2.py index dd32403..236f2ea 100644 --- a/pato2.py +++ b/pato2.py @@ -26,10 +26,9 @@ """ from repm.config import * -import tarfile, commands +import tarfile from glob import glob -from user import home -from os.path import isdir, isfile, realpath +from os.path import isdir, isfile def printf(text,output_=output): """Guarda el texto en la variable log y puede imprimir en pantalla.""" -- cgit v1.2.3-2-g168b From 80402467cf38f32373ac14d992d2daee4736ad82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 00:42:33 -0600 Subject: Added primitive testcase --- test/directory_list | 2 ++ test/link_list | 2 ++ test/package_list | 3 +++ test/test1.py | 22 ++++++++++++++++++++++ 4 files changed, 29 insertions(+) create mode 100644 test/directory_list create mode 100644 test/link_list create mode 100644 test/package_list create mode 100644 test/test1.py diff --git a/test/directory_list b/test/directory_list new file mode 100644 index 0000000..c0f7f47 --- /dev/null +++ b/test/directory_list @@ -0,0 +1,2 @@ +drwxrwxr-x 15 2010/09/11 11:28:50 community-staging +drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os \ No newline at end of file diff --git a/test/link_list b/test/link_list new file mode 100644 index 0000000..d015364 --- /dev/null +++ b/test/link_list @@ -0,0 +1,2 @@ +lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz +lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz \ No newline at end of file diff --git a/test/package_list b/test/package_list new file mode 100644 index 0000000..6ffa1de --- /dev/null +++ b/test/package_list @@ -0,0 +1,3 @@ +-rw-rw-r-- 5846249 2010/11/13 10:54:25 pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz +-rw-rw-r-- 982768 2011/02/05 14:38:17 pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz +-rw-rw-r-- 982764 2011/02/05 14:38:40 pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz \ No newline at end of file diff --git a/test/test1.py b/test/test1.py new file mode 100644 index 0000000..8e5cd1e --- /dev/null +++ b/test/test1.py @@ -0,0 +1,22 @@ +""" """ + +__author__ = "Joshua Ismael Haase Hernández " +__version__ = "$Revision: 1.1 $" +__date__ = "$Date: 2011/02/08 $" +__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" +__license__ = "GPL3+" + +import repm.filter +import unittest +import commands + +class KnownValues(unittest.TestCase): + + def testDirectoryOutput(self): + """get_file_list_from_rsync_output should ignore directories""" + output=commands.getoutput("cat ./directory_list") + result=get_file_list_from_rsync_output(output) + self.assertEqual(tuple(), result) + +if __name__ == "__main__": + unittest.main() -- cgit v1.2.3-2-g168b From 0bdbdeaf186a9ab73f596b60748b6ae8a25ebaeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 01:18:18 -0600 Subject: Added some testcases. --- test/test1.py | 48 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/test/test1.py b/test/test1.py index 8e5cd1e..a42e9a5 100644 --- a/test/test1.py +++ b/test/test1.py @@ -6,17 +6,59 @@ __date__ = "$Date: 2011/02/08 $" __copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" __license__ = "GPL3+" -import repm.filter +from repm.config import * +from repm.filter import * import unittest -import commands class KnownValues(unittest.TestCase): + directory_list=("drwxrwxr-x 15 2010/09/11 11:28:50 community-staging", + "drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os") + # (output, name, version, arch) + link_list=( + ("lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz", "apvlv","0.1.0","i686"), + ("lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz","calibre","0.7.44","i686"), + ) + package_list=( + ("-rw-rw-r-- 5846249 2010/11/13 10:54:25 pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz", + "abuse","0.7.1","x86_64"), + ("-rw-rw-r-- 982768 2011/02/05 14:38:17 pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz", + "acetoneiso2","2.3","i686"), + ("-rw-rw-r-- 982764 2011/02/05 14:38:40 pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz", + "acetoneiso2","2.3","x86_64") + ) + + def generate_results(example_tuple): + a=list() + for output, name, version, arch in example_tuple: + pkg=Packages() + pkg["name"] = name + pkg["version"] = version + pkg["arch"] = arch + a.append(pkg) + return tuple(a) def testDirectoryOutput(self): """get_file_list_from_rsync_output should ignore directories""" - output=commands.getoutput("cat ./directory_list") + rsync_out="\n".join(directory_list) result=get_file_list_from_rsync_output(output) self.assertEqual(tuple(), result) + def testLinkOutput(self): + """get_file_list_from_rsync_output should make a Package Object + from links """ + correct_result=generate_results(link_list) + rsync_out="\n".join([a for a,b,c,d in link_list]) + result=get_file_list_from_rsync_output(rsync_out) + self.assertEqual(correct_result, result) + + + def testPackageOutput(self): + """get_file_list_from_rsync_output should make a Package Object + from links """ + correct_result=generate_results(package_list) + rsync_out="\n".join([a for a,b,c,d in package_list]) + result=get_file_list_from_rsync_output(rsync_out) + self.assertEqual(correct_result, result) + if __name__ == "__main__": unittest.main() -- cgit v1.2.3-2-g168b From 3633ea3c4ec93436b2344dba79b06ff00c7ae528 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 01:30:35 -0600 Subject: Corrected first testcases. --- test/test1.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/test1.py b/test/test1.py index a42e9a5..495ba05 100644 --- a/test/test1.py +++ b/test/test1.py @@ -1,3 +1,4 @@ +# -*- encoding: utf-8 -*- """ """ __author__ = "Joshua Ismael Haase Hernández " @@ -27,10 +28,10 @@ class KnownValues(unittest.TestCase): "acetoneiso2","2.3","x86_64") ) - def generate_results(example_tuple): + def generate_results(self, example_tuple): a=list() for output, name, version, arch in example_tuple: - pkg=Packages() + pkg=Package() pkg["name"] = name pkg["version"] = version pkg["arch"] = arch @@ -39,24 +40,23 @@ class KnownValues(unittest.TestCase): def testDirectoryOutput(self): """get_file_list_from_rsync_output should ignore directories""" - rsync_out="\n".join(directory_list) + rsync_out="\n".join(self.directory_list) result=get_file_list_from_rsync_output(output) self.assertEqual(tuple(), result) def testLinkOutput(self): """get_file_list_from_rsync_output should make a Package Object from links """ - correct_result=generate_results(link_list) - rsync_out="\n".join([a for a,b,c,d in link_list]) + correct_result=self.generate_results(self.link_list) + rsync_out="\n".join([a for a,b,c,d in self.link_list]) result=get_file_list_from_rsync_output(rsync_out) self.assertEqual(correct_result, result) - def testPackageOutput(self): """get_file_list_from_rsync_output should make a Package Object from links """ - correct_result=generate_results(package_list) - rsync_out="\n".join([a for a,b,c,d in package_list]) + correct_result=self.generate_results(self.package_list) + rsync_out="\n".join([a for a,b,c,d in self.package_list]) result=get_file_list_from_rsync_output(rsync_out) self.assertEqual(correct_result, result) -- cgit v1.2.3-2-g168b From 61f5ecfa6b57f31c14430a7de5099640fcbe3525 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 03:13:39 -0600 Subject: * Updated Package class * Corrected test1.py -> for get_package_list_from_rsync_output --- config.py | 1 + test/test1.py | 22 ++++++++++++---------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/config.py b/config.py index 6dcd348..a7087c4 100644 --- a/config.py +++ b/config.py @@ -49,6 +49,7 @@ class Package: """ An object that has information about a package. """ package_info={ "name" : False, "version" : False, + "release" : False, "arch" : False, "license" : False, "location": False} diff --git a/test/test1.py b/test/test1.py index 495ba05..db87f51 100644 --- a/test/test1.py +++ b/test/test1.py @@ -14,41 +14,43 @@ import unittest class KnownValues(unittest.TestCase): directory_list=("drwxrwxr-x 15 2010/09/11 11:28:50 community-staging", "drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os") - # (output, name, version, arch) + # (output, name, version, arch, release, location) link_list=( - ("lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz", "apvlv","0.1.0","i686"), - ("lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz","calibre","0.7.44","i686"), + ("lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz", "apvlv","0.1.0","i686", "2", "community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz"), + ("lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz","calibre","0.7.44","i686", "2", "community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz"), ) package_list=( ("-rw-rw-r-- 5846249 2010/11/13 10:54:25 pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz", - "abuse","0.7.1","x86_64"), + "abuse","0.7.1","x86_64","1","pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz"), ("-rw-rw-r-- 982768 2011/02/05 14:38:17 pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz", - "acetoneiso2","2.3","i686"), + "acetoneiso2","2.3","i686", "2", "pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz"), ("-rw-rw-r-- 982764 2011/02/05 14:38:40 pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz", - "acetoneiso2","2.3","x86_64") + "acetoneiso2","2.3","x86_64","2","pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz") ) def generate_results(self, example_tuple): a=list() - for output, name, version, arch in example_tuple: + for output, name, version, arch, release, location in example_tuple: pkg=Package() pkg["name"] = name pkg["version"] = version pkg["arch"] = arch + pkg["release"] = release + pkg["location"] = location a.append(pkg) return tuple(a) def testDirectoryOutput(self): """get_file_list_from_rsync_output should ignore directories""" rsync_out="\n".join(self.directory_list) - result=get_file_list_from_rsync_output(output) + result=get_file_list_from_rsync_output(rsync_out) self.assertEqual(tuple(), result) def testLinkOutput(self): """get_file_list_from_rsync_output should make a Package Object from links """ correct_result=self.generate_results(self.link_list) - rsync_out="\n".join([a for a,b,c,d in self.link_list]) + rsync_out="\n".join([a for a,b,c,d,e,f in self.link_list]) result=get_file_list_from_rsync_output(rsync_out) self.assertEqual(correct_result, result) @@ -56,7 +58,7 @@ class KnownValues(unittest.TestCase): """get_file_list_from_rsync_output should make a Package Object from links """ correct_result=self.generate_results(self.package_list) - rsync_out="\n".join([a for a,b,c,d in self.package_list]) + rsync_out="\n".join([a for a,b,c,d,e,f in self.package_list]) result=get_file_list_from_rsync_output(rsync_out) self.assertEqual(correct_result, result) -- cgit v1.2.3-2-g168b From 8fcf2f7f622de21b580ee808de53cffcc9200639 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 03:41:12 -0600 Subject: Better testcases --- test/test1.py | 60 +++++++++++++++++++++++++++++++---------------------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/test/test1.py b/test/test1.py index db87f51..4ec72d3 100644 --- a/test/test1.py +++ b/test/test1.py @@ -14,12 +14,10 @@ import unittest class KnownValues(unittest.TestCase): directory_list=("drwxrwxr-x 15 2010/09/11 11:28:50 community-staging", "drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os") - # (output, name, version, arch, release, location) - link_list=( + # (rsync_out, name, version, arch, release, location) + examples=( ("lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz", "apvlv","0.1.0","i686", "2", "community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz"), ("lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz","calibre","0.7.44","i686", "2", "community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz"), - ) - package_list=( ("-rw-rw-r-- 5846249 2010/11/13 10:54:25 pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz", "abuse","0.7.1","x86_64","1","pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz"), ("-rw-rw-r-- 982768 2011/02/05 14:38:17 pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz", @@ -29,16 +27,8 @@ class KnownValues(unittest.TestCase): ) def generate_results(self, example_tuple): - a=list() - for output, name, version, arch, release, location in example_tuple: - pkg=Package() - pkg["name"] = name - pkg["version"] = version - pkg["arch"] = arch - pkg["release"] = release - pkg["location"] = location - a.append(pkg) - return tuple(a) + rsync_out="\n".join([a for a,b,c,d,e,f in example_tuple]) + return get_file_list_from_rsync_output(rsync_out) def testDirectoryOutput(self): """get_file_list_from_rsync_output should ignore directories""" @@ -46,21 +36,35 @@ class KnownValues(unittest.TestCase): result=get_file_list_from_rsync_output(rsync_out) self.assertEqual(tuple(), result) - def testLinkOutput(self): - """get_file_list_from_rsync_output should make a Package Object - from links """ - correct_result=self.generate_results(self.link_list) - rsync_out="\n".join([a for a,b,c,d,e,f in self.link_list]) - result=get_file_list_from_rsync_output(rsync_out) - self.assertEqual(correct_result, result) + def testNames(self): + results=self.generate_results(self.examples) + var =[name for rsync_out, name, version, arch, release, location in self.examples] + for i in range(len(results)): + self.assertEqual(results[i]["name"], var[i]) - def testPackageOutput(self): - """get_file_list_from_rsync_output should make a Package Object - from links """ - correct_result=self.generate_results(self.package_list) - rsync_out="\n".join([a for a,b,c,d,e,f in self.package_list]) - result=get_file_list_from_rsync_output(rsync_out) - self.assertEqual(correct_result, result) + def testVersions(self): + results=self.generate_results(self.examples) + var = [version for rsync_out, name, version, arch, release, location in self.examples] + for i in range(len(results)): + self.assertEqual(results[i]["name"], var[i]) + + def testArchs(self): + results=self.generate_results(self.examples) + var = [arch for rsync_out, name, version, arch, release, location in self.examples] + for i in range(len(results)): + self.assertEqual(results[i]["name"], var[i]) + + def testReleases(self): + results=self.generate_results(self.examples) + var = [release for rsync_out, name, version, arch, release, location in self.examples] + for i in range(len(results)): + self.assertEqual(results[i]["name"], var[i]) + + def testLocations(self): + results=self.generate_results(self.examples) + var = [location for rsync_out, name, version, arch, release, location in self.examples] + for i in range(len(results)): + self.assertEqual(results[i]["name"], var[i]) if __name__ == "__main__": unittest.main() -- cgit v1.2.3-2-g168b From 748ef10d89e5898c1ebffb08165f18cd3829119e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 03:43:07 -0600 Subject: Fixed stupid error. --- test/test1.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test1.py b/test/test1.py index 4ec72d3..ea488e9 100644 --- a/test/test1.py +++ b/test/test1.py @@ -46,25 +46,25 @@ class KnownValues(unittest.TestCase): results=self.generate_results(self.examples) var = [version for rsync_out, name, version, arch, release, location in self.examples] for i in range(len(results)): - self.assertEqual(results[i]["name"], var[i]) + self.assertEqual(results[i]["version"], var[i]) def testArchs(self): results=self.generate_results(self.examples) var = [arch for rsync_out, name, version, arch, release, location in self.examples] for i in range(len(results)): - self.assertEqual(results[i]["name"], var[i]) + self.assertEqual(results[i]["arch"], var[i]) def testReleases(self): results=self.generate_results(self.examples) var = [release for rsync_out, name, version, arch, release, location in self.examples] for i in range(len(results)): - self.assertEqual(results[i]["name"], var[i]) + self.assertEqual(results[i]["release"], var[i]) def testLocations(self): results=self.generate_results(self.examples) var = [location for rsync_out, name, version, arch, release, location in self.examples] for i in range(len(results)): - self.assertEqual(results[i]["name"], var[i]) + self.assertEqual(results[i]["location"], var[i]) if __name__ == "__main__": unittest.main() -- cgit v1.2.3-2-g168b From df79a07c092880259a851d4ebf9febe9a8e19880 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 10:47:55 -0600 Subject: Fixed test1.py --- test/test1.py | 41 ++++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/test/test1.py b/test/test1.py index ea488e9..61b0651 100644 --- a/test/test1.py +++ b/test/test1.py @@ -26,9 +26,9 @@ class KnownValues(unittest.TestCase): "acetoneiso2","2.3","x86_64","2","pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz") ) - def generate_results(self, example_tuple): - rsync_out="\n".join([a for a,b,c,d,e,f in example_tuple]) - return get_file_list_from_rsync_output(rsync_out) + def generate_results(self, example_tuple, attr): + rsync_out, name, version, arch, release, location = example_tuple + return get_file_list_from_rsync_output(rsync_out)[0][attr], locals()[attr] def testDirectoryOutput(self): """get_file_list_from_rsync_output should ignore directories""" @@ -37,34 +37,29 @@ class KnownValues(unittest.TestCase): self.assertEqual(tuple(), result) def testNames(self): - results=self.generate_results(self.examples) - var =[name for rsync_out, name, version, arch, release, location in self.examples] - for i in range(len(results)): - self.assertEqual(results[i]["name"], var[i]) + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="name") + self.assertEqual(k, v) def testVersions(self): - results=self.generate_results(self.examples) - var = [version for rsync_out, name, version, arch, release, location in self.examples] - for i in range(len(results)): - self.assertEqual(results[i]["version"], var[i]) + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="version") + self.assertEqual(k, v) def testArchs(self): - results=self.generate_results(self.examples) - var = [arch for rsync_out, name, version, arch, release, location in self.examples] - for i in range(len(results)): - self.assertEqual(results[i]["arch"], var[i]) + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="arch") + self.assertEqual(k, v) def testReleases(self): - results=self.generate_results(self.examples) - var = [release for rsync_out, name, version, arch, release, location in self.examples] - for i in range(len(results)): - self.assertEqual(results[i]["release"], var[i]) + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="release") + self.assertEqual(k, v) def testLocations(self): - results=self.generate_results(self.examples) - var = [location for rsync_out, name, version, arch, release, location in self.examples] - for i in range(len(results)): - self.assertEqual(results[i]["location"], var[i]) + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="location") + self.assertEqual(k, v) if __name__ == "__main__": unittest.main() -- cgit v1.2.3-2-g168b From 9c8b6615d537a9c7bb483457cd8a300d7f9c597e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 10:55:33 -0600 Subject: * Added filter.py * Added __init__.py for test module * Removed unused files --- filter.py | 90 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ test/__init__.py | 0 test/link_list | 2 -- test/package_list | 3 -- 4 files changed, 90 insertions(+), 5 deletions(-) create mode 100644 filter.py create mode 100644 test/__init__.py delete mode 100644 test/link_list delete mode 100644 test/package_list diff --git a/filter.py b/filter.py new file mode 100644 index 0000000..5229e63 --- /dev/null +++ b/filter.py @@ -0,0 +1,90 @@ +#! /usr/bin/python +#-*- encoding: utf-8 -*- +import commands +import os +import re +from repm.config import * +from repm.pato2 import * + +rsync_list_command="rsync -av --no-motd --list-only " + +def generate_rsync_command(base_command, dir_list, destdir=repodir, mirror_name=mirror, + mirror_path=mirrorpath, blacklist_file=False): + """ Generates an rsync command for executing it by combining all parameters. + + Parameters: + ---------- + base_command -> str + mirror_name -> str + mirror_path -> str + dir_list -> list or tuple + destdir -> str Must be a dir + blacklist_file -> False or str File must exist + + Return: + ---------- + rsync_command -> str """ + from os.path import isfile, isdir + + if blacklist_file and not isfile(blacklist_file): + print(blacklist_file + " is not a file") + raise NonValidFile + + if not os.path.isdir(destdir): + print(destdir + " is not a directory") + raise NonValidDir + + dir_list="{" + ",".join(dir_list) + "}" + + if blacklist_file: + return " ".join((base_command, "--exclude-from-file="+blacklist_file, + mirror_name + mirror_path + dir_list, destdir)) + return " ".join((base_command, mirror_name + mirror_path + dir_list, destdir)) + +def run_rsync(base_for_rsync=rsync_list_command, dir_list_for_rsync=(repo_list + dir_list), + debug=verbose): + """ Runs rsync and gets returns it's output """ + cmd = str(generate_rsync_command(rsync_list_command, (repo_list + dir_list))) + if debug: + printf("rsync_command" + cmd) + return commands.getoutput(cmd) + +def get_file_list_from_rsync_output(rsync_output): + """ Generates a list of packages and versions from an rsync output using --list-only --no-motd. + + Parameters: + ---------- + rsync_output -> str containing output from rsync + + Returns: + ---------- + package_list -> tuple of Package objects. """ + a=list() + + def directory(line): + pass + + def package_or_link(line): + """ Take info out of filename """ + location_field = 4 + pkg = Package() + pkg["location"] = line.rsplit()[location_field] + fileattrs = pkg["location"].split("/")[-1].split("-") + pkg["arch"] = fileattrs.pop(-1).split(".")[0] + pkg["release"] = fileattrs.pop(-1) + pkg["version"] = fileattrs.pop(-1) + pkg["name"] = "-".join(fileattrs) + return pkg + + options = { "d": directory, + "l": package_or_link, + "-": package_or_link} + + for line in rsync_output.split("\n"): + if ".pkg.tar.gz" or ".pkg.tar.xz" in line: + pkginfo=options[line[0]](line) + if pkginfo: + a.append(pkginfo) + + return tuple(a) + diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/link_list b/test/link_list deleted file mode 100644 index d015364..0000000 --- a/test/link_list +++ /dev/null @@ -1,2 +0,0 @@ -lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz -lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz \ No newline at end of file diff --git a/test/package_list b/test/package_list deleted file mode 100644 index 6ffa1de..0000000 --- a/test/package_list +++ /dev/null @@ -1,3 +0,0 @@ --rw-rw-r-- 5846249 2010/11/13 10:54:25 pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz --rw-rw-r-- 982768 2011/02/05 14:38:17 pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz --rw-rw-r-- 982764 2011/02/05 14:38:40 pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz \ No newline at end of file -- cgit v1.2.3-2-g168b From 1c178e144ae404c6ee332e25c9f3ae37fb9abb23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 10:56:31 -0600 Subject: * Modified gitignore for ignoring pyc files --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e4e5f6c..e645833 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -*~ \ No newline at end of file +*~ +*.pyc \ No newline at end of file -- cgit v1.2.3-2-g168b From 5f08ca1edd02121f950a829745a512a2bc58b016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 11:05:26 -0600 Subject: * Changed variable name for consistency. --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index a7087c4..6206607 100644 --- a/config.py +++ b/config.py @@ -38,7 +38,7 @@ verbose = False blacklist = docs + "/blacklist.txt" whitelist = docs + "/whitelist.txt" pending = docs + "/pending" -rsyncBlacklist = docs + "/rsyncBlacklist" +rsync_blacklist = docs + "/rsyncBlacklist" # Classes and Exceptions class NonValidFile(ValueError): pass -- cgit v1.2.3-2-g168b From d9ac9d17a92607b09fb9afa91ff96aeae38dbdf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 11:55:25 -0600 Subject: Delete unneeded file. --- test/directory_list | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 test/directory_list diff --git a/test/directory_list b/test/directory_list deleted file mode 100644 index c0f7f47..0000000 --- a/test/directory_list +++ /dev/null @@ -1,2 +0,0 @@ -drwxrwxr-x 15 2010/09/11 11:28:50 community-staging -drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os \ No newline at end of file -- cgit v1.2.3-2-g168b From 2cae61ff561561e405f0cd0c150dbcd77ce25b82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 13:26:54 -0600 Subject: config py: * Make Package class init, restrict keys. filter.py * Corrected function * Added generate_rsync_exclude --- config.py | 31 ++++++++++++++++++++++++------- filter.py | 53 ++++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 68 insertions(+), 16 deletions(-) diff --git a/config.py b/config.py index 6206607..bd8ef1b 100644 --- a/config.py +++ b/config.py @@ -47,15 +47,21 @@ class NonValidCommand(ValueError): pass class Package: """ An object that has information about a package. """ - package_info={ "name" : False, - "version" : False, - "release" : False, - "arch" : False, - "license" : False, - "location": False} + package_info=dict() + def __init__(self): + self.package_info={ "name" : False, + "version" : False, + "release" : False, + "arch" : False, + "license" : False, + "location": False} + def __setitem__(self, key, item): - return self.package_info.__setitem__(key, item) + if key in self.package_info.keys(): + return self.package_info.__setitem__(key, item) + else: + raise ValueError("Package has no %s attribute" % key) def __getitem__(self, key): return self.package_info.__getitem__(key) @@ -63,4 +69,15 @@ class Package: def __unicode__(self): return str(self.package_info) + def __repr__(self): + return str(self.package_info) + + def __eq__(self,x): + if not isinstance(x, Package): + return False + for key in self.package_info.keys(): + if x[key] != self[key]: + return False + else: + return True diff --git a/filter.py b/filter.py index 5229e63..817a228 100644 --- a/filter.py +++ b/filter.py @@ -6,7 +6,7 @@ import re from repm.config import * from repm.pato2 import * -rsync_list_command="rsync -av --no-motd --list-only " +rsync_list_command="rsync -a --no-motd --list-only " def generate_rsync_command(base_command, dir_list, destdir=repodir, mirror_name=mirror, mirror_path=mirrorpath, blacklist_file=False): @@ -18,8 +18,8 @@ def generate_rsync_command(base_command, dir_list, destdir=repodir, mirror_name= mirror_name -> str mirror_path -> str dir_list -> list or tuple - destdir -> str Must be a dir - blacklist_file -> False or str File must exist + destdir -> str Path to dir, dir must exist. + blacklist_file -> False or str Path to file, file must exist. Return: ---------- @@ -54,11 +54,11 @@ def get_file_list_from_rsync_output(rsync_output): Parameters: ---------- - rsync_output -> str containing output from rsync + rsync_output -> str Contains output from rsync Returns: ---------- - package_list -> tuple of Package objects. """ + package_list -> tuple Contains Package objects. """ a=list() def directory(line): @@ -81,10 +81,45 @@ def get_file_list_from_rsync_output(rsync_output): "-": package_or_link} for line in rsync_output.split("\n"): - if ".pkg.tar.gz" or ".pkg.tar.xz" in line: - pkginfo=options[line[0]](line) - if pkginfo: - a.append(pkginfo) + if ".pkg.tar" in line: + pkginfo=options[line[0]](line) + if pkginfo: + a.append(pkginfo) return tuple(a) +def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, + blacklist_file=rsync_blacklist, debug=verbose): + """ Generate an exclude list for rsync + + Parameters: + ---------- + package_iterable -> list or tuple Contains Package objects + blacklisted_names-> list or tuple Contains blacklisted names + blacklist_file -> str Path to file + debug -> bool + + Output: + ---------- + if debug == False -> None + if debug == True -> blacklist """ + a=list() + + for package in packages_iterable: + if not isinstance(package, Package): + raise ValueError(" %s is not a Package object " % package) + if package["name"] in blacklisted_names: + a.append(package["location"]) + + if debug: + printf(a) + + try: + fsock = open(blacklist_file,"w") + try: + fsock.write("\n".join(a)) + finally: + fsock.close() + except IOError: + printf("%s wasnt written" % blacklist_file) + -- cgit v1.2.3-2-g168b From 072787627a2339c3d5aca541086c2e4545bbad8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 8 Feb 2011 14:06:58 -0600 Subject: * Changed variable name in filter.py * Added example in test1.py --- filter.py | 15 +++++++-------- test/test1.py | 5 +++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/filter.py b/filter.py index 817a228..ffc0965 100644 --- a/filter.py +++ b/filter.py @@ -89,20 +89,19 @@ def get_file_list_from_rsync_output(rsync_output): return tuple(a) def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, - blacklist_file=rsync_blacklist, debug=verbose): + exclude_file=rsync_blacklist, debug=verbose): """ Generate an exclude list for rsync Parameters: ---------- - package_iterable -> list or tuple Contains Package objects - blacklisted_names-> list or tuple Contains blacklisted names - blacklist_file -> str Path to file - debug -> bool + package_iterable -> list or tuple Contains Package objects + blacklisted_names-> list or tuple Contains blacklisted names + exclude_file -> str Path to file + debug -> bool If True, file list gets logged Output: ---------- - if debug == False -> None - if debug == True -> blacklist """ + None """ a=list() for package in packages_iterable: @@ -115,7 +114,7 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, printf(a) try: - fsock = open(blacklist_file,"w") + fsock = open(exclude_file,"w") try: fsock.write("\n".join(a)) finally: diff --git a/test/test1.py b/test/test1.py index 61b0651..13b5660 100644 --- a/test/test1.py +++ b/test/test1.py @@ -13,7 +13,8 @@ import unittest class KnownValues(unittest.TestCase): directory_list=("drwxrwxr-x 15 2010/09/11 11:28:50 community-staging", - "drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os") + "drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os", + 'dr-xr-sr-x 4096 2010/09/11 11:37:10 .') # (rsync_out, name, version, arch, release, location) examples=( ("lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz", "apvlv","0.1.0","i686", "2", "community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz"), @@ -60,6 +61,6 @@ class KnownValues(unittest.TestCase): for i in self.examples: k,v = self.generate_results(example_tuple=i,attr="location") self.assertEqual(k, v) - + if __name__ == "__main__": unittest.main() -- cgit v1.2.3-2-g168b From 6a37d25e51228a0a83d004b9acda63c528972a26 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 13 Feb 2011 14:26:40 -0800 Subject: Fixed glob expression --- pato2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pato2.py b/pato2.py index 236f2ea..cfdd859 100644 --- a/pato2.py +++ b/pato2.py @@ -54,7 +54,7 @@ def packages(repo_, arch_, expr="*"): def sync_all_repo(verbose_=verbose): folders = ",".join(repo_list + dir_list) - cmd_ = "rsync -av --progress --delete-after --delay-updates " + mirror + mirrorpath + "/{" + folders + "} " + repodir + cmd_ = "rsync -av --delete-after --delay-updates " + mirror + mirrorpath + "/{" + folders + "} " + repodir printf(cmd_) a=commands.getoutput(cmd_) if verbose_: printf(a) @@ -144,11 +144,11 @@ def add_free_repo(verbose_=verbose): for repo_ in repo_list: for arch_ in arch_list: lista_=list() - for file_ in glob(free_path + repo_ + "/os/" + arch_ + "/*"): + for file_ in glob(free_path + repo_ + "/os/" + arch_ + "/*.pkg.tar.*"): lista_.append(file_) link(repo_,arch_,file_) for dir_ in other: - for file_ in glob(free_path + repo_ + "/os/" + dir_ + "/*"): + for file_ in glob(free_path + repo_ + "/os/" + dir_ + "/*.pkg.tar.*"): lista_.append(file_) link(repo_,arch_,file_) if lista_: -- cgit v1.2.3-2-g168b From 2321a53d35a9736b8c5c715ca40a9af69b1bf64e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 01:14:47 -0600 Subject: Rearranged functions --- config.py | 5 +++++ filter.py | 43 ------------------------------------------- pato2.py | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 43 deletions(-) diff --git a/config.py b/config.py index bd8ef1b..4854e0d 100644 --- a/config.py +++ b/config.py @@ -40,6 +40,11 @@ whitelist = docs + "/whitelist.txt" pending = docs + "/pending" rsync_blacklist = docs + "/rsyncBlacklist" +# Rsync commands + +rsync_list_command="rsync -a --no-motd --list-only " + + # Classes and Exceptions class NonValidFile(ValueError): pass class NonValidDir(ValueError): pass diff --git a/filter.py b/filter.py index ffc0965..b51dba8 100644 --- a/filter.py +++ b/filter.py @@ -6,49 +6,6 @@ import re from repm.config import * from repm.pato2 import * -rsync_list_command="rsync -a --no-motd --list-only " - -def generate_rsync_command(base_command, dir_list, destdir=repodir, mirror_name=mirror, - mirror_path=mirrorpath, blacklist_file=False): - """ Generates an rsync command for executing it by combining all parameters. - - Parameters: - ---------- - base_command -> str - mirror_name -> str - mirror_path -> str - dir_list -> list or tuple - destdir -> str Path to dir, dir must exist. - blacklist_file -> False or str Path to file, file must exist. - - Return: - ---------- - rsync_command -> str """ - from os.path import isfile, isdir - - if blacklist_file and not isfile(blacklist_file): - print(blacklist_file + " is not a file") - raise NonValidFile - - if not os.path.isdir(destdir): - print(destdir + " is not a directory") - raise NonValidDir - - dir_list="{" + ",".join(dir_list) + "}" - - if blacklist_file: - return " ".join((base_command, "--exclude-from-file="+blacklist_file, - mirror_name + mirror_path + dir_list, destdir)) - return " ".join((base_command, mirror_name + mirror_path + dir_list, destdir)) - -def run_rsync(base_for_rsync=rsync_list_command, dir_list_for_rsync=(repo_list + dir_list), - debug=verbose): - """ Runs rsync and gets returns it's output """ - cmd = str(generate_rsync_command(rsync_list_command, (repo_list + dir_list))) - if debug: - printf("rsync_command" + cmd) - return commands.getoutput(cmd) - def get_file_list_from_rsync_output(rsync_output): """ Generates a list of packages and versions from an rsync output using --list-only --no-motd. diff --git a/pato2.py b/pato2.py index cfdd859..125ae8c 100644 --- a/pato2.py +++ b/pato2.py @@ -166,6 +166,46 @@ def get_licenses(verbose_=verbose): a=commands.getoutput(cmd_) if verbose_: printf(a) +def generate_rsync_command(base_command, dir_list, destdir=repodir, + source=mirror+mirrorpath, blacklist_file=False): + """ Generates an rsync command for executing it by combining all parameters. + + Parameters: + ---------- + base_command -> str + dir_list -> list or tuple + destdir -> str Path to dir, dir must exist. + source -> str The source for rsync + blacklist_file -> False or str Path to file, file must exist. + + Return: + ---------- + rsync_command -> str """ + from os.path import isfile, isdir + + if blacklist_file and not isfile(blacklist_file): + print(blacklist_file + " is not a file") + raise NonValidFile + + if not os.path.isdir(destdir): + print(destdir + " is not a directory") + raise NonValidDir + + dir_list="{" + ",".join(dir_list) + "}" + + if blacklist_file: + return " ".join((base_command, "--exclude-from-file="+blacklist_file, + mirror_name + mirror_path + dir_list, destdir)) + return " ".join((base_command, mirror_name + mirror_path + dir_list, destdir)) + +def run_rsync(base_for_rsync=rsync_list_command, dir_list_for_rsync=(repo_list + dir_list), + debug=verbose): + """ Runs rsync and gets returns it's output """ + cmd = str(generate_rsync_command(rsync_list_command, (repo_list + dir_list))) + if debug: + printf("rsync_command" + cmd) + return commands.getoutput(cmd) + if __name__ == "__main__": from time import time start_time = time() -- cgit v1.2.3-2-g168b From 6203dc2fc926781db694ca2383b1e44e2c5469c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 01:29:33 -0600 Subject: - filter.py: * Makes a blacklist for rsync - pato2.py: * Has more functions --- filter.py | 6 +++++- pato2.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/filter.py b/filter.py index b51dba8..14ae31e 100644 --- a/filter.py +++ b/filter.py @@ -1,4 +1,4 @@ -#! /usr/bin/python + #! /usr/bin/python #-*- encoding: utf-8 -*- import commands import os @@ -79,3 +79,7 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, except IOError: printf("%s wasnt written" % blacklist_file) +if name == "__main__": + a=run_rsync(rsync_list_command) + packages=get_file_list_from_rsync_output(a) + generate_exclude_list_from_blacklist(packages,listado(blacklist)) diff --git a/pato2.py b/pato2.py index 125ae8c..97eec9d 100644 --- a/pato2.py +++ b/pato2.py @@ -198,7 +198,7 @@ def generate_rsync_command(base_command, dir_list, destdir=repodir, mirror_name + mirror_path + dir_list, destdir)) return " ".join((base_command, mirror_name + mirror_path + dir_list, destdir)) -def run_rsync(base_for_rsync=rsync_list_command, dir_list_for_rsync=(repo_list + dir_list), +def run_rsync(base_for_rsync, dir_list_for_rsync=(repo_list + dir_list), debug=verbose): """ Runs rsync and gets returns it's output """ cmd = str(generate_rsync_command(rsync_list_command, (repo_list + dir_list))) -- cgit v1.2.3-2-g168b From b27cbc08447fe5605bf877ee0991888d5ecf6382 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 01:32:48 -0600 Subject: Corrected filter.py to run --- filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filter.py b/filter.py index 14ae31e..c789cd5 100644 --- a/filter.py +++ b/filter.py @@ -79,7 +79,7 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, except IOError: printf("%s wasnt written" % blacklist_file) -if name == "__main__": +if __name__ == "__main__": a=run_rsync(rsync_list_command) packages=get_file_list_from_rsync_output(a) generate_exclude_list_from_blacklist(packages,listado(blacklist)) -- cgit v1.2.3-2-g168b From 931daf34033714e64fb4f98a101dd804e4e6363e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 01:34:35 -0600 Subject: Added os module to config file --- config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/config.py b/config.py index 4854e0d..bd46688 100644 --- a/config.py +++ b/config.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- from user import home import commands +import os time__ = commands.getoutput("date +%Y%m%d-%H:%M") -- cgit v1.2.3-2-g168b From 7f40b89e8cd0e9b2cfd92a8b74c4063d8b9aa3e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 01:38:59 -0600 Subject: Corrected funcions --- pato2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pato2.py b/pato2.py index 97eec9d..55e63e7 100644 --- a/pato2.py +++ b/pato2.py @@ -195,13 +195,13 @@ def generate_rsync_command(base_command, dir_list, destdir=repodir, if blacklist_file: return " ".join((base_command, "--exclude-from-file="+blacklist_file, - mirror_name + mirror_path + dir_list, destdir)) - return " ".join((base_command, mirror_name + mirror_path + dir_list, destdir)) + source + dir_list, destdir)) + return " ".join((base_command, source + dir_list, destdir)) def run_rsync(base_for_rsync, dir_list_for_rsync=(repo_list + dir_list), debug=verbose): """ Runs rsync and gets returns it's output """ - cmd = str(generate_rsync_command(rsync_list_command, (repo_list + dir_list))) + cmd = str(generate_rsync_command(base_for_rsync, (repo_list + dir_list))) if debug: printf("rsync_command" + cmd) return commands.getoutput(cmd) -- cgit v1.2.3-2-g168b From cbe877f2ba25b398ad32b92d22dc6f5a108ef59f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 02:28:01 -0600 Subject: * Changed get_file_list_ for pkginfo in function names * Added pkginfo_from_... funcions --- filter.py | 57 ++++++++++++++++++++++++++++++++++++++------------------- test/test1.py | 8 ++++---- 2 files changed, 42 insertions(+), 23 deletions(-) diff --git a/filter.py b/filter.py index c789cd5..232319f 100644 --- a/filter.py +++ b/filter.py @@ -1,13 +1,34 @@ #! /usr/bin/python #-*- encoding: utf-8 -*- import commands -import os import re from repm.config import * from repm.pato2 import * -def get_file_list_from_rsync_output(rsync_output): - """ Generates a list of packages and versions from an rsync output using --list-only --no-motd. +def pkginfo_from_filename(filename): + """ Generates a Package object with info from a filename, + filename can be relative or absolute + + Parameters: + ---------- + filename -> str + + Returns: + ---------- + pkg -> Package object""" + pkg = Package() + pkg["location"] = filename + fileattrs = os.path.basename(filename).split("-") + pkg["arch"] = fileattrs.pop(-1).split(".")[0] + pkg["release"] = fileattrs.pop(-1) + pkg["version"] = fileattrs.pop(-1) + pkg["name"] = "-".join(fileattrs) + return pkg + + +def pkginfo_from_rsync_output(rsync_output): + """ Generates a list of packages and versions from an rsync output + wich uses --list-only and --no-motd options. Parameters: ---------- @@ -18,33 +39,30 @@ def get_file_list_from_rsync_output(rsync_output): package_list -> tuple Contains Package objects. """ a=list() - def directory(line): - pass - def package_or_link(line): """ Take info out of filename """ location_field = 4 - pkg = Package() - pkg["location"] = line.rsplit()[location_field] - fileattrs = pkg["location"].split("/")[-1].split("-") - pkg["arch"] = fileattrs.pop(-1).split(".")[0] - pkg["release"] = fileattrs.pop(-1) - pkg["version"] = fileattrs.pop(-1) - pkg["name"] = "-".join(fileattrs) - return pkg - + pkginfo_from_filename(line.rsplit()[location_field]) + + def directory(line): + pass + options = { "d": directory, "l": package_or_link, "-": package_or_link} for line in rsync_output.split("\n"): if ".pkg.tar" in line: - pkginfo=options[line[0]](line) - if pkginfo: - a.append(pkginfo) + pkginfo_=options[line[0]](line) + if pkginfo_: + a.append(pkginfo_) return tuple(a) +def pkginfo_from_files_in_dir(directory): + """ Returns pkginfo from filenames of packages in dir + wich has .pkg.tar.{xz,gz} on them """ + def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, exclude_file=rsync_blacklist, debug=verbose): """ Generate an exclude list for rsync @@ -78,8 +96,9 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, fsock.close() except IOError: printf("%s wasnt written" % blacklist_file) + if __name__ == "__main__": a=run_rsync(rsync_list_command) - packages=get_file_list_from_rsync_output(a) + packages=pkginfo_from_rsync_output(a) generate_exclude_list_from_blacklist(packages,listado(blacklist)) diff --git a/test/test1.py b/test/test1.py index 13b5660..9e94352 100644 --- a/test/test1.py +++ b/test/test1.py @@ -29,12 +29,12 @@ class KnownValues(unittest.TestCase): def generate_results(self, example_tuple, attr): rsync_out, name, version, arch, release, location = example_tuple - return get_file_list_from_rsync_output(rsync_out)[0][attr], locals()[attr] + return pkginfo_from_rsync_output(rsync_out)[0][attr], locals()[attr] def testDirectoryOutput(self): - """get_file_list_from_rsync_output should ignore directories""" + """pkginfo_from_rsync_output should ignore directories""" rsync_out="\n".join(self.directory_list) - result=get_file_list_from_rsync_output(rsync_out) + result=pkginfo_from_rsync_output(rsync_out) self.assertEqual(tuple(), result) def testNames(self): @@ -62,5 +62,5 @@ class KnownValues(unittest.TestCase): k,v = self.generate_results(example_tuple=i,attr="location") self.assertEqual(k, v) -if __name__ == "__main__": +if __name__ == "__mpain__": unittest.main() -- cgit v1.2.3-2-g168b From 4e7508d9a61c3653b46da8fa513513756bb3b6f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 16:46:33 -0600 Subject: * pkginfo functions --- filter.py | 44 +++++++++++++++++++++++++++++++------------- test/test1.py | 2 +- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/filter.py b/filter.py index 232319f..ad3b33d 100644 --- a/filter.py +++ b/filter.py @@ -1,7 +1,6 @@ #! /usr/bin/python #-*- encoding: utf-8 -*- -import commands -import re +import glob from repm.config import * from repm.pato2 import * @@ -11,11 +10,13 @@ def pkginfo_from_filename(filename): Parameters: ---------- - filename -> str + filename -> str Must contain .pkg.tar. Returns: ---------- pkg -> Package object""" + if ".pkg.tar." not in filename: + raise NonValidFile pkg = Package() pkg["location"] = filename fileattrs = os.path.basename(filename).split("-") @@ -37,31 +38,49 @@ def pkginfo_from_rsync_output(rsync_output): Returns: ---------- package_list -> tuple Contains Package objects. """ - a=list() + package_list=list() def package_or_link(line): """ Take info out of filename """ location_field = 4 pkginfo_from_filename(line.rsplit()[location_field]) - def directory(line): + def do_nothing(): pass - - options = { "d": directory, + + options = { "d": do_nothing, "l": package_or_link, - "-": package_or_link} + "-": package_or_link, + " ": do_nothing} for line in rsync_output.split("\n"): if ".pkg.tar" in line: - pkginfo_=options[line[0]](line) + pkginfo=options[line[0]](line) if pkginfo_: - a.append(pkginfo_) + package_list.append(pkginfo) - return tuple(a) + return tuple(package_list) def pkginfo_from_files_in_dir(directory): """ Returns pkginfo from filenames of packages in dir - wich has .pkg.tar.{xz,gz} on them """ + wich has .pkg.tar. on them + + Parameters: + ---------- + directory -> str Directory must exist + + Returns: + ---------- + package_list -> tuple Contains Package objects """ + package_list=list() + + if not os.path.isdir(directory): + raise NonValidDir + + for filename in glob(os.path.join(directory,"*")): + if ".pkg.tar." in filename: + package_list.append(pkginfo_from_filename(filename)) + return tuple(package_list) def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, exclude_file=rsync_blacklist, debug=verbose): @@ -97,7 +116,6 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, except IOError: printf("%s wasnt written" % blacklist_file) - if __name__ == "__main__": a=run_rsync(rsync_list_command) packages=pkginfo_from_rsync_output(a) diff --git a/test/test1.py b/test/test1.py index 9e94352..b810e9a 100644 --- a/test/test1.py +++ b/test/test1.py @@ -62,5 +62,5 @@ class KnownValues(unittest.TestCase): k,v = self.generate_results(example_tuple=i,attr="location") self.assertEqual(k, v) -if __name__ == "__mpain__": +if __name__ == "__main__": unittest.main() -- cgit v1.2.3-2-g168b From 1db8ccbc9221be6c30884172c00caefa217f66a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 16:47:03 -0600 Subject: Added test for pkginfo_from_file --- test/test_pkginfo_from_file.py | 76 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 test/test_pkginfo_from_file.py diff --git a/test/test_pkginfo_from_file.py b/test/test_pkginfo_from_file.py new file mode 100644 index 0000000..0d7ede6 --- /dev/null +++ b/test/test_pkginfo_from_file.py @@ -0,0 +1,76 @@ +#! /usr/bin/python +# -*- encoding: utf-8 -*- +""" """ + +__author__ = "Joshua Ismael Haase Hernández " +__version__ = "$Revision: 1.1 $" +__date__ = "$Date: 2011/02/08 $" +__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" +__license__ = "GPL3+" + +from repm.config import * +from repm.filter import * +import unittest + +class KnownValues(unittest.TestCase): + # (filename, name, version, release, arch) + # filename is location + known=( + ("community-testing/os/i686/inputattach-1.24-3-i686.pkg.tar.xz","inputattach","1.24","3","i686"), + ("community-testing/os/i686/ngspice-22-1-i686.pkg.tar.xz","ngspice","22","1","i686"), + ("community-testing/os/i686/tmux-1.4-2-i686.pkg.tar.xz","tmux","1.4","2","i686"), + ("community-testing/os/i686/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), + ("../../../pool/community/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), + ("community-testing/os/x86_64/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), + ("../../../pool/community/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), + ("tor-0.2.1.29-2-x86_64.pkg.tar.xz","tor","0.2.1.29","2","x86_64"), + ) + + def generate_results(self, example_tuple, attr): + location, name, version, release, arch = example_tuple + return pkginfo_from_filename(location)[attr], locals()[attr] + + def testReturnPackageObject(self): + for i in self.known: + location, name, version, release, arch = i + self.assertIsInstance(pkginfo_from_filename(location),Package) + + def testNames(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="name") + self.assertEqual(k, v) + + def testVersions(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="version") + self.assertEqual(k, v) + + def testArchs(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="arch") + self.assertEqual(k, v) + + def testReleases(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="release") + self.assertEqual(k, v) + + def testLocations(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="location") + self.assertEqual(k, v) + +class BadInput(unittest.TestCase): + bad=("community-testing/os/i686/community-testing.db", + "community-testing/os/i686/community-testing.db.tar.gz", + "community-testing/os/i686/community-testing.db.tar.gz.old", + "community-testing/os/i686/community-testing.files", + "community-testing/os/i686/community-testing.files.tar.gz", + "community-testing/os/x86_64") + + def testBadInput(self): + for i in self.bad: + self.assertRaises(NonValidFile,pkginfo_from_filename,i) + +if __name__ == "__main__": + unittest.main() -- cgit v1.2.3-2-g168b From b525eeca86d3c5a25f971a12ceeae44fadd6fa94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 16:59:28 -0600 Subject: renamed test1 to be informative --- test/test_pkginfo_from_rsync_output.py | 66 ++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 test/test_pkginfo_from_rsync_output.py diff --git a/test/test_pkginfo_from_rsync_output.py b/test/test_pkginfo_from_rsync_output.py new file mode 100644 index 0000000..b810e9a --- /dev/null +++ b/test/test_pkginfo_from_rsync_output.py @@ -0,0 +1,66 @@ +# -*- encoding: utf-8 -*- +""" """ + +__author__ = "Joshua Ismael Haase Hernández " +__version__ = "$Revision: 1.1 $" +__date__ = "$Date: 2011/02/08 $" +__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" +__license__ = "GPL3+" + +from repm.config import * +from repm.filter import * +import unittest + +class KnownValues(unittest.TestCase): + directory_list=("drwxrwxr-x 15 2010/09/11 11:28:50 community-staging", + "drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os", + 'dr-xr-sr-x 4096 2010/09/11 11:37:10 .') + # (rsync_out, name, version, arch, release, location) + examples=( + ("lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz", "apvlv","0.1.0","i686", "2", "community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz"), + ("lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz","calibre","0.7.44","i686", "2", "community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz"), + ("-rw-rw-r-- 5846249 2010/11/13 10:54:25 pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz", + "abuse","0.7.1","x86_64","1","pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz"), + ("-rw-rw-r-- 982768 2011/02/05 14:38:17 pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz", + "acetoneiso2","2.3","i686", "2", "pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz"), + ("-rw-rw-r-- 982764 2011/02/05 14:38:40 pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz", + "acetoneiso2","2.3","x86_64","2","pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz") + ) + + def generate_results(self, example_tuple, attr): + rsync_out, name, version, arch, release, location = example_tuple + return pkginfo_from_rsync_output(rsync_out)[0][attr], locals()[attr] + + def testDirectoryOutput(self): + """pkginfo_from_rsync_output should ignore directories""" + rsync_out="\n".join(self.directory_list) + result=pkginfo_from_rsync_output(rsync_out) + self.assertEqual(tuple(), result) + + def testNames(self): + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="name") + self.assertEqual(k, v) + + def testVersions(self): + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="version") + self.assertEqual(k, v) + + def testArchs(self): + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="arch") + self.assertEqual(k, v) + + def testReleases(self): + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="release") + self.assertEqual(k, v) + + def testLocations(self): + for i in self.examples: + k,v = self.generate_results(example_tuple=i,attr="location") + self.assertEqual(k, v) + +if __name__ == "__main__": + unittest.main() -- cgit v1.2.3-2-g168b From a20a7fc689fd0e8680cbc13adfa6fdc2849e4397 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 17:00:13 -0600 Subject: idem --- test/test1.py | 66 ----------------------------------------------------------- 1 file changed, 66 deletions(-) delete mode 100644 test/test1.py diff --git a/test/test1.py b/test/test1.py deleted file mode 100644 index b810e9a..0000000 --- a/test/test1.py +++ /dev/null @@ -1,66 +0,0 @@ -# -*- encoding: utf-8 -*- -""" """ - -__author__ = "Joshua Ismael Haase Hernández " -__version__ = "$Revision: 1.1 $" -__date__ = "$Date: 2011/02/08 $" -__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" -__license__ = "GPL3+" - -from repm.config import * -from repm.filter import * -import unittest - -class KnownValues(unittest.TestCase): - directory_list=("drwxrwxr-x 15 2010/09/11 11:28:50 community-staging", - "drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os", - 'dr-xr-sr-x 4096 2010/09/11 11:37:10 .') - # (rsync_out, name, version, arch, release, location) - examples=( - ("lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz", "apvlv","0.1.0","i686", "2", "community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz"), - ("lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz","calibre","0.7.44","i686", "2", "community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz"), - ("-rw-rw-r-- 5846249 2010/11/13 10:54:25 pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz", - "abuse","0.7.1","x86_64","1","pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz"), - ("-rw-rw-r-- 982768 2011/02/05 14:38:17 pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz", - "acetoneiso2","2.3","i686", "2", "pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz"), - ("-rw-rw-r-- 982764 2011/02/05 14:38:40 pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz", - "acetoneiso2","2.3","x86_64","2","pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz") - ) - - def generate_results(self, example_tuple, attr): - rsync_out, name, version, arch, release, location = example_tuple - return pkginfo_from_rsync_output(rsync_out)[0][attr], locals()[attr] - - def testDirectoryOutput(self): - """pkginfo_from_rsync_output should ignore directories""" - rsync_out="\n".join(self.directory_list) - result=pkginfo_from_rsync_output(rsync_out) - self.assertEqual(tuple(), result) - - def testNames(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="name") - self.assertEqual(k, v) - - def testVersions(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="version") - self.assertEqual(k, v) - - def testArchs(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="arch") - self.assertEqual(k, v) - - def testReleases(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="release") - self.assertEqual(k, v) - - def testLocations(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="location") - self.assertEqual(k, v) - -if __name__ == "__main__": - unittest.main() -- cgit v1.2.3-2-g168b From 9fecb003deaae2364e0d9d41055ca33a7d1a2131 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 17:09:16 -0600 Subject: * Changed format of run_rsync debug output. --- pato2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pato2.py b/pato2.py index 55e63e7..13242f7 100644 --- a/pato2.py +++ b/pato2.py @@ -203,7 +203,7 @@ def run_rsync(base_for_rsync, dir_list_for_rsync=(repo_list + dir_list), """ Runs rsync and gets returns it's output """ cmd = str(generate_rsync_command(base_for_rsync, (repo_list + dir_list))) if debug: - printf("rsync_command" + cmd) + printf("rsync_command: " + cmd) return commands.getoutput(cmd) if __name__ == "__main__": -- cgit v1.2.3-2-g168b From e4ffbeff4bbdeb3f57c39c6f6dac3ce902db9b4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 17:14:37 -0600 Subject: * Fixed generate_rsync_command --- pato2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pato2.py b/pato2.py index 13242f7..74d18be 100644 --- a/pato2.py +++ b/pato2.py @@ -195,8 +195,8 @@ def generate_rsync_command(base_command, dir_list, destdir=repodir, if blacklist_file: return " ".join((base_command, "--exclude-from-file="+blacklist_file, - source + dir_list, destdir)) - return " ".join((base_command, source + dir_list, destdir)) + os.join(source, dir_list), destdir)) + return " ".join((base_command, os.join(source, dir_list), destdir)) def run_rsync(base_for_rsync, dir_list_for_rsync=(repo_list + dir_list), debug=verbose): -- cgit v1.2.3-2-g168b From e5b869eade3fbf47cf9ac1b7c6be34dfee14b2c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 17:17:31 -0600 Subject: * Fixed generate_rsync_command --- pato2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pato2.py b/pato2.py index 74d18be..e32a4dd 100644 --- a/pato2.py +++ b/pato2.py @@ -195,8 +195,8 @@ def generate_rsync_command(base_command, dir_list, destdir=repodir, if blacklist_file: return " ".join((base_command, "--exclude-from-file="+blacklist_file, - os.join(source, dir_list), destdir)) - return " ".join((base_command, os.join(source, dir_list), destdir)) + os.path.join(source, dir_list), destdir)) + return " ".join((base_command, os.path.join(source, dir_list), destdir)) def run_rsync(base_for_rsync, dir_list_for_rsync=(repo_list + dir_list), debug=verbose): -- cgit v1.2.3-2-g168b From f923ab304017a71136642ed7b9780441edcaf441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 19:36:26 -0600 Subject: * TestCase for pkginfo_from_rsync_output * Corrected __eq__ method in Package class * Corrected pkginfo_from_rsync_output --- config.py | 2 +- filter.py | 18 ++++---- test/rsync_output_sample | 14 +++++++ test/test_pkginfo_from_rsync_output.py | 76 ++++++++++++---------------------- 4 files changed, 52 insertions(+), 58 deletions(-) create mode 100644 test/rsync_output_sample diff --git a/config.py b/config.py index bd46688..465a8a8 100644 --- a/config.py +++ b/config.py @@ -82,7 +82,7 @@ class Package: if not isinstance(x, Package): return False for key in self.package_info.keys(): - if x[key] != self[key]: + if x[key] != self.package_info[key]: return False else: return True diff --git a/filter.py b/filter.py index ad3b33d..55ab94a 100644 --- a/filter.py +++ b/filter.py @@ -38,26 +38,28 @@ def pkginfo_from_rsync_output(rsync_output): Returns: ---------- package_list -> tuple Contains Package objects. """ - package_list=list() def package_or_link(line): """ Take info out of filename """ location_field = 4 - pkginfo_from_filename(line.rsplit()[location_field]) + return pkginfo_from_filename(line.rsplit()[location_field]) def do_nothing(): - pass + """""" options = { "d": do_nothing, "l": package_or_link, "-": package_or_link, " ": do_nothing} + + package_list=list() - for line in rsync_output.split("\n"): - if ".pkg.tar" in line: - pkginfo=options[line[0]](line) - if pkginfo_: - package_list.append(pkginfo) + lines=[x for x in rsync_output.split("\n") if ".pkg.tar" in x] + + for line in lines: + pkginfo=options[line[0]](line) + if pkginfo: + package_list.append(pkginfo) return tuple(package_list) diff --git a/test/rsync_output_sample b/test/rsync_output_sample new file mode 100644 index 0000000..72d9cd0 --- /dev/null +++ b/test/rsync_output_sample @@ -0,0 +1,14 @@ +dr-xr-sr-x 4096 2010/09/11 11:37:10 . +-rw-r--r-- 11 2011/02/08 00:00:01 lastsync +drwxrwxr-x 15 2010/09/11 11:28:50 community-staging +drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os +drwxrwxr-x 8192 2011/02/07 17:00:01 community-staging/os/i686 +lrwxrwxrwx 52 2010/12/23 16:51:01 community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz -> ../../../pool/community/alex-2.3.4-1-i686.pkg.tar.xz +lrwxrwxrwx 27 2011/02/07 14:02:54 community-staging/os/i686/community-staging.db -> community-staging.db.tar.gz +-rw-rw-r-- 2237 2011/02/07 14:02:54 community-staging/os/i686/community-staging.db.tar.gz +-rw-rw-r-- 3209 2011/02/07 14:00:13 community-staging/os/i686/community-staging.db.tar.gz.old +drwxrwxr-x 15 2009/07/22 15:07:56 community +drwxrwxr-x 40 2009/08/04 15:57:42 community/os +drwxrwsr-x 36864 2011/02/03 05:00:01 community/os/any +-rw-rw-r-- 303336 2010/07/16 10:06:28 community/os/any/any2dvd-0.34-4-any.pkg.tar.xz +-rw-rw-r-- 221664 2010/03/28 15:55:48 community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz diff --git a/test/test_pkginfo_from_rsync_output.py b/test/test_pkginfo_from_rsync_output.py index b810e9a..81f14d9 100644 --- a/test/test_pkginfo_from_rsync_output.py +++ b/test/test_pkginfo_from_rsync_output.py @@ -12,55 +12,33 @@ from repm.filter import * import unittest class KnownValues(unittest.TestCase): - directory_list=("drwxrwxr-x 15 2010/09/11 11:28:50 community-staging", - "drwxrwxr-x 30 2010/09/11 11:28:50 community-staging/os", - 'dr-xr-sr-x 4096 2010/09/11 11:37:10 .') - # (rsync_out, name, version, arch, release, location) - examples=( - ("lrwxrwxrwx 53 2011/01/31 01:52:06 community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz -> ../../../pool/community/apvlv-0.1.0-2-i686.pkg.tar.xz", "apvlv","0.1.0","i686", "2", "community-testing/os/i686/apvlv-0.1.0-2-i686.pkg.tar.xz"), - ("lrwxrwxrwx 56 2011/02/04 14:34:08 community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz -> ../../../pool/community/calibre-0.7.44-2-i686.pkg.tar.xz","calibre","0.7.44","i686", "2", "community-testing/os/i686/calibre-0.7.44-2-i686.pkg.tar.xz"), - ("-rw-rw-r-- 5846249 2010/11/13 10:54:25 pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz", - "abuse","0.7.1","x86_64","1","pool/community/abuse-0.7.1-1-x86_64.pkg.tar.gz"), - ("-rw-rw-r-- 982768 2011/02/05 14:38:17 pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz", - "acetoneiso2","2.3","i686", "2", "pool/community/acetoneiso2-2.3-2-i686.pkg.tar.xz"), - ("-rw-rw-r-- 982764 2011/02/05 14:38:40 pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz", - "acetoneiso2","2.3","x86_64","2","pool/community/acetoneiso2-2.3-2-x86_64.pkg.tar.xz") - ) - - def generate_results(self, example_tuple, attr): - rsync_out, name, version, arch, release, location = example_tuple - return pkginfo_from_rsync_output(rsync_out)[0][attr], locals()[attr] - - def testDirectoryOutput(self): - """pkginfo_from_rsync_output should ignore directories""" - rsync_out="\n".join(self.directory_list) - result=pkginfo_from_rsync_output(rsync_out) - self.assertEqual(tuple(), result) - - def testNames(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="name") - self.assertEqual(k, v) - - def testVersions(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="version") - self.assertEqual(k, v) - - def testArchs(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="arch") - self.assertEqual(k, v) - - def testReleases(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="release") - self.assertEqual(k, v) - - def testLocations(self): - for i in self.examples: - k,v = self.generate_results(example_tuple=i,attr="location") - self.assertEqual(k, v) + try: + output_file = open("rsync_output_sample") + rsync_out= output_file.read() + output_file.close() + except IOError: print("There is no rsync_output_sample file") + + pkglist = pkginfo_from_rsync_output(rsync_out) + + def testOutputArePackages(self): + if not self.pkglist: + self.fail("not pkglist:" + str(self.pkglist)) + for pkg in self.pkglist: + self.assertIsInstance(pkg,Package) + + def testFirstPkg(self): + first_package_info=Package() + first_package_info.package_info={ "name" : "alex", + "version" : "2.3.4", + "release" : "1", + "arch" : "i686", + "license" : False, + "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz"} + if self.pkglist: + first_package=self.pkglist[0] + else: + self.fail(self.pkglist) + self.assertEqual(first_package,first_package_info) if __name__ == "__main__": unittest.main() -- cgit v1.2.3-2-g168b From 3866f148dae1f6c90fc6d903784b65e452c048c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 22:45:11 -0600 Subject: * Added debug action to generate_exclude_list_from_blacklist * Added test data to test_pkginfo_from_rsync_output.py --- filter.py | 2 +- test/test_pkginfo_from_rsync_output.py | 38 ++++++++++++++++++++++++---------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/filter.py b/filter.py index 55ab94a..f8182e3 100644 --- a/filter.py +++ b/filter.py @@ -108,7 +108,7 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, if debug: printf(a) - + return a try: fsock = open(exclude_file,"w") try: diff --git a/test/test_pkginfo_from_rsync_output.py b/test/test_pkginfo_from_rsync_output.py index 81f14d9..5ed5dd8 100644 --- a/test/test_pkginfo_from_rsync_output.py +++ b/test/test_pkginfo_from_rsync_output.py @@ -11,7 +11,27 @@ from repm.config import * from repm.filter import * import unittest -class KnownValues(unittest.TestCase): +example_package_list=(Package(),Package(),Package()) +example_package_list[0].package_info={ "name" : "alex", + "version" : "2.3.4", + "release" : "1", + "arch" : "i686", + "license" : False, + "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz"} +example_package_list[1].package_info={ "name" : "any2dvd", + "version" : "0.34", + "release" : "4", + "arch" : "any", + "license" : False, + "location": "community/os/any/any2dvd-0.34-4-any.pkg.tar.xz"} +example_package_list[2].package_info={ "name" : "gmime22", + "version" : "2.2.26", + "release" : "1", + "arch" : "x86_64", + "license" : False, + "location": "community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz"} + +class pkginfoFromRsyncOutput(unittest.TestCase): try: output_file = open("rsync_output_sample") rsync_out= output_file.read() @@ -26,19 +46,15 @@ class KnownValues(unittest.TestCase): for pkg in self.pkglist: self.assertIsInstance(pkg,Package) - def testFirstPkg(self): - first_package_info=Package() - first_package_info.package_info={ "name" : "alex", - "version" : "2.3.4", - "release" : "1", - "arch" : "i686", - "license" : False, - "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz"} + def testPackageInfo(self): if self.pkglist: first_package=self.pkglist[0] else: self.fail(self.pkglist) - self.assertEqual(first_package,first_package_info) - + self.assertEqual(first_package,example_package_list[0]) + +class generateRsyncBlacklist(unittest.TestCase): + """ Test Blacklist generation """ + if __name__ == "__main__": unittest.main() -- cgit v1.2.3-2-g168b From 9de298a7b88f7f36aec4c9344356a935c67cfeb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 14 Feb 2011 23:27:46 -0600 Subject: * Added test for generate_exclude_list_from_blacklist * Fixed listado to strip spaces --- filter.py | 1 - pato2.py | 2 +- test/blacklist_sample | 2 ++ test/test_pkginfo_from_rsync_output.py | 17 +++++++++++------ 4 files changed, 14 insertions(+), 8 deletions(-) create mode 100644 test/blacklist_sample diff --git a/filter.py b/filter.py index f8182e3..f91ec68 100644 --- a/filter.py +++ b/filter.py @@ -107,7 +107,6 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, a.append(package["location"]) if debug: - printf(a) return a try: fsock = open(exclude_file,"w") diff --git a/pato2.py b/pato2.py index e32a4dd..5bc97a9 100644 --- a/pato2.py +++ b/pato2.py @@ -42,7 +42,7 @@ def listado(filename_): archivo = open(filename_,"r") lista = archivo.read().split("\n") archivo.close() - return [pkg.split(":")[0] for pkg in lista if pkg] + return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] def db(repo_,arch_): """Construye un nombre para sincronizar una base de datos.""" diff --git a/test/blacklist_sample b/test/blacklist_sample new file mode 100644 index 0000000..2a02af6 --- /dev/null +++ b/test/blacklist_sample @@ -0,0 +1,2 @@ +alex:alex-libre: Aquí va un comentario +gmime22 ::Non free dependencies \ No newline at end of file diff --git a/test/test_pkginfo_from_rsync_output.py b/test/test_pkginfo_from_rsync_output.py index 5ed5dd8..aca49e8 100644 --- a/test/test_pkginfo_from_rsync_output.py +++ b/test/test_pkginfo_from_rsync_output.py @@ -47,14 +47,19 @@ class pkginfoFromRsyncOutput(unittest.TestCase): self.assertIsInstance(pkg,Package) def testPackageInfo(self): - if self.pkglist: - first_package=self.pkglist[0] - else: - self.fail(self.pkglist) - self.assertEqual(first_package,example_package_list[0]) + if not self.pkglist: + self.fail("Pkglist doesn't exist: " + str(self.pkglist)) + self.assertEqual(self.pkglist,example_package_list) class generateRsyncBlacklist(unittest.TestCase): """ Test Blacklist generation """ - + def testListado(self): + self.assertEqual(listado("blacklist_sample"),["alex","gmime22"]) + + def testExcludeFiles(self): + a=generate_exclude_list_from_blacklist(example_package_list,listado("blacklist_sample"),debug=True) + b=[example_package_list[0]["location"],example_package_list[2]["location"]] + self.assertEqual(a,b) + if __name__ == "__main__": unittest.main() -- cgit v1.2.3-2-g168b From cf3253774c64e6430d4e3afb826cc6c8b0e4b91b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 15 Feb 2011 00:13:37 -0600 Subject: * Upgraded sync_all_repo --- config.py | 2 +- pato2.py | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/config.py b/config.py index 465a8a8..36b54df 100644 --- a/config.py +++ b/config.py @@ -44,7 +44,7 @@ rsync_blacklist = docs + "/rsyncBlacklist" # Rsync commands rsync_list_command="rsync -a --no-motd --list-only " - +rsync_update_command="rsync -av --delete-after --delay-updates " # Classes and Exceptions class NonValidFile(ValueError): pass diff --git a/pato2.py b/pato2.py index 5bc97a9..9b902f4 100644 --- a/pato2.py +++ b/pato2.py @@ -25,7 +25,7 @@ """ from repm.config import * - +from repm.filter import * import tarfile from glob import glob from os.path import isdir, isfile @@ -52,12 +52,14 @@ def packages(repo_, arch_, expr="*"): """ Get packages on a repo, arch folder """ return tuple( glob( repodir + "/" + repo_ + "/os/" + arch_ + "/" + expr ) ) -def sync_all_repo(verbose_=verbose): - folders = ",".join(repo_list + dir_list) - cmd_ = "rsync -av --delete-after --delay-updates " + mirror + mirrorpath + "/{" + folders + "} " + repodir - printf(cmd_) - a=commands.getoutput(cmd_) - if verbose_: printf(a) +def sync_all_repo(debug=verbose): + cmd=generate_rsync_command(rsync_list_command) + rsout=run_rsync(cmd) + pkgs=pkginfo_from_rsync_output(rsout) + generate_exclude_list_from_blacklist(pkgs,listado(blacklist)) + cmd=generate_rsync_command(rsync_update_command,blacklist_file=blacklist) + a=run_rsync(cmd) + if debug: printf(a) def get_from_desc(desc, var,db_tar_file=False): """ Get a var from desc file """ @@ -166,7 +168,7 @@ def get_licenses(verbose_=verbose): a=commands.getoutput(cmd_) if verbose_: printf(a) -def generate_rsync_command(base_command, dir_list, destdir=repodir, +def generate_rsync_command(base_command, dir_list=(repo_list + dir_list), destdir=repodir, source=mirror+mirrorpath, blacklist_file=False): """ Generates an rsync command for executing it by combining all parameters. @@ -198,10 +200,8 @@ def generate_rsync_command(base_command, dir_list, destdir=repodir, os.path.join(source, dir_list), destdir)) return " ".join((base_command, os.path.join(source, dir_list), destdir)) -def run_rsync(base_for_rsync, dir_list_for_rsync=(repo_list + dir_list), - debug=verbose): +def run_rsync(command,debug=verbose): """ Runs rsync and gets returns it's output """ - cmd = str(generate_rsync_command(base_for_rsync, (repo_list + dir_list))) if debug: printf("rsync_command: " + cmd) return commands.getoutput(cmd) -- cgit v1.2.3-2-g168b From 8975a9021c8b04fb81d9b9358a55b89ecdb373b7 Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 14 Feb 2011 22:18:36 -0800 Subject: changed config --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index 36b54df..c60e8f3 100644 --- a/config.py +++ b/config.py @@ -33,7 +33,7 @@ other = ("any",) # Output output = True -verbose = False +verbose = True # Files blacklist = docs + "/blacklist.txt" -- cgit v1.2.3-2-g168b From fb247388925beabe01162f28bfcba7fcf8809254 Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 14 Feb 2011 22:41:29 -0800 Subject: * Fixed for rsync to run --- pato2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pato2.py b/pato2.py index 9b902f4..c598c2f 100644 --- a/pato2.py +++ b/pato2.py @@ -196,15 +196,15 @@ def generate_rsync_command(base_command, dir_list=(repo_list + dir_list), destdi dir_list="{" + ",".join(dir_list) + "}" if blacklist_file: - return " ".join((base_command, "--exclude-from-file="+blacklist_file, + return " ".join((base_command, "--exclude-from="+blacklist_file, os.path.join(source, dir_list), destdir)) return " ".join((base_command, os.path.join(source, dir_list), destdir)) def run_rsync(command,debug=verbose): """ Runs rsync and gets returns it's output """ if debug: - printf("rsync_command: " + cmd) - return commands.getoutput(cmd) + printf("rsync_command: " + command) + return commands.getoutput(command) if __name__ == "__main__": from time import time -- cgit v1.2.3-2-g168b From 7bb59afd65871ed2949b86ec6cef805ea2fd3dd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Tue, 15 Feb 2011 19:25:51 -0800 Subject: Fixed add_free_repo --- config.py | 1 + pato2.py | 295 ++++++++++++++++++++++++++++++++------------------------------ 2 files changed, 152 insertions(+), 144 deletions(-) diff --git a/config.py b/config.py index c60e8f3..6420355 100644 --- a/config.py +++ b/config.py @@ -19,6 +19,7 @@ logdir = path + "/log" ## Must be defined logname= logdir + "/" + time__ + "-repo-maintainer.log" +freedir= path + "/free/" repodir= path + "/repo" tmp = home + "/tmp" archdb = tmp + "/db" diff --git a/pato2.py b/pato2.py index c598c2f..f5deb48 100644 --- a/pato2.py +++ b/pato2.py @@ -31,142 +31,149 @@ from glob import glob from os.path import isdir, isfile def printf(text,output_=output): - """Guarda el texto en la variable log y puede imprimir en pantalla.""" - log_file = open(logname, 'a') - log_file.write("\n" + str(text) + "\n") - log_file.close() - if output_: print (str(text) + "\n") + """Guarda el texto en la variable log y puede imprimir en pantalla.""" + log_file = open(logname, 'a') + log_file.write("\n" + str(text) + "\n") + log_file.close() + if output_: print (str(text) + "\n") def listado(filename_): - """Obtiene una lista de paquetes de un archivo.""" - archivo = open(filename_,"r") - lista = archivo.read().split("\n") - archivo.close() - return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] + """Obtiene una lista de paquetes de un archivo.""" + archivo = open(filename_,"r") + lista = archivo.read().split("\n") + archivo.close() + return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] def db(repo_,arch_): - """Construye un nombre para sincronizar una base de datos.""" - return "/%s/os/%s/%s.db.tar.gz" % (repo_, arch_, repo_) + """Construye un nombre para sincronizar una base de datos.""" + return "/%s/os/%s/%s.db.tar.gz" % (repo_, arch_, repo_) def packages(repo_, arch_, expr="*"): - """ Get packages on a repo, arch folder """ - return tuple( glob( repodir + "/" + repo_ + "/os/" + arch_ + "/" + expr ) ) + """ Get packages on a repo, arch folder """ + return tuple( glob( repodir + "/" + repo_ + "/os/" + arch_ + "/" + expr ) ) def sync_all_repo(debug=verbose): - cmd=generate_rsync_command(rsync_list_command) - rsout=run_rsync(cmd) - pkgs=pkginfo_from_rsync_output(rsout) - generate_exclude_list_from_blacklist(pkgs,listado(blacklist)) - cmd=generate_rsync_command(rsync_update_command,blacklist_file=blacklist) - a=run_rsync(cmd) - if debug: printf(a) + cmd=generate_rsync_command(rsync_list_command) + rsout=run_rsync(cmd) + pkgs=pkginfo_from_rsync_output(rsout) + generate_exclude_list_from_blacklist(pkgs,listado(blacklist)) + cmd=generate_rsync_command(rsync_update_command,blacklist_file=blacklist) + a=run_rsync(cmd) + if debug: printf(a) def get_from_desc(desc, var,db_tar_file=False): - """ Get a var from desc file """ - desc = desc.split("\n") - return desc[desc.index(var)+1] + """ Get a var from desc file """ + desc = desc.split("\n") + return desc[desc.index(var)+1] def get_info(repo_,arch_,db_tar_file=False,verbose_=verbose): - """ Makes a list of package name, file and license """ - info=list() - # Extract DB tar.gz - commands.getoutput("mkdir -p " + archdb) - if not db_tar_file: - db_tar_file = repodir + db(repo_,arch_) - if isfile(db_tar_file): - try: - db_open_tar = tarfile.open(db_tar_file, 'r:gz') - except tarfile.ReadError: - printf("No valid db_file %s" % db_tar_file) - return(tuple()) - else: - printf("No db_file %s" % db_tar_file) - return(tuple()) - for file in db_open_tar.getmembers(): - db_open_tar.extract(file, archdb) - db_open_tar.close() - # Get info from file - for dir_ in glob(archdb + "/*"): - if isdir(dir_) and isfile(dir_ + "/desc"): - pkg_desc_file = open(dir_ + "/desc", "r") - desc = pkg_desc_file.read() - pkg_desc_file.close() - info.append(( get_from_desc(desc,"%NAME%"), - dir_.split("/")[-1], - get_from_desc(desc,"%LICENSE%") )) - if verbose_: printf(info) - commands.getoutput("rm -r %s/*" % archdb) - return tuple(info) + """ Makes a list of package name, file and license """ + info=list() + # Extract DB tar.gz + commands.getoutput("mkdir -p " + archdb) + if not db_tar_file: + db_tar_file = repodir + db(repo_,arch_) + if isfile(db_tar_file): + try: + db_open_tar = tarfile.open(db_tar_file, 'r:gz') + except tarfile.ReadError: + printf("No valid db_file %s" % db_tar_file) + return(tuple()) + else: + printf("No db_file %s" % db_tar_file) + return(tuple()) + for file in db_open_tar.getmembers(): + db_open_tar.extract(file, archdb) + db_open_tar.close() + # Get info from file + for dir_ in glob(archdb + "/*"): + if isdir(dir_) and isfile(dir_ + "/desc"): + pkg_desc_file = open(dir_ + "/desc", "r") + desc = pkg_desc_file.read() + pkg_desc_file.close() + info.append(( get_from_desc(desc,"%NAME%"), + dir_.split("/")[-1], + get_from_desc(desc,"%LICENSE%") )) + if verbose_: printf(info) + commands.getoutput("rm -r %s/*" % archdb) + return tuple(info) def make_pending(repo_,arch_,info_): - """ Si los paquetes no están en blacklist ni whitelist y la licencia contiene "custom" los agrega a pending""" - search = tuple( listado(blacklist) + listado (whitelist) ) - if verbose: printf("blaclist + whitelist= " + str(search) ) - lista_=list() - for (name,pkg_,license_) in info_: - if "custom" in license_: - if name not in search: - lista_.append( (name, license_ ) ) - elif not name: - printf( pkg_ + " package has no %NAME% attibute " ) - if verbose: printf( lista_ ) - a=open( pending + "-" + repo_ + ".txt", "w" ).write( - "\n".join([name + ":" + license_ for (name,license_) in lista_]) ) + """ Si los paquetes no están en blacklist ni whitelist y la licencia contiene "custom" los agrega a pending""" + search = tuple( listado(blacklist) + listado (whitelist) ) + if verbose: printf("blaclist + whitelist= " + str(search) ) + lista_=list() + for (name,pkg_,license_) in info_: + if "custom" in license_: + if name not in search: + lista_.append( (name, license_ ) ) + elif not name: + printf( pkg_ + " package has no %NAME% attibute " ) + if verbose: printf( lista_ ) + a=open( pending + "-" + repo_ + ".txt", "w" ).write( + "\n".join([name + ":" + license_ for (name,license_) in lista_]) ) def remove_from_blacklist(repo_,arch_,info_,blacklist_): - """ Check the blacklist and remove packages on the db""" - lista_=list() - pack_=list() - for (name_, pkg_, license_) in info_: - if name_ in blacklist_: - lista_.append(name_) - for p in packages(repo_,arch_,pkg_ + "*"): - pack_.append(p) - if lista_: - lista_=" ".join(lista_) - com_ = "repo-remove " + repodir + db(repo_,arch_) + " " + lista_ - printf(com_) - a = commands.getoutput(com_) - if verbose: printf(a) - if pack_: - pack_=" ".join(pack_) - com_="chmod a-r " + pack_ - printf(com_) - a=commands.getoutput(com_) - if verbose: printf(a) + """ Check the blacklist and remove packages on the db""" + lista_=list() + pack_=list() + for (name_, pkg_, license_) in info_: + if name_ in blacklist_: + lista_.append(name_) + for p in packages(repo_,arch_,pkg_ + "*"): + pack_.append(p) + if lista_: + lista_=" ".join(lista_) + com_ = "repo-remove " + repodir + db(repo_,arch_) + " " + lista_ + printf(com_) + a = commands.getoutput(com_) + if verbose: printf(a) + if pack_: + pack_=" ".join(pack_) + com_="chmod a-r " + pack_ + printf(com_) + a=commands.getoutput(com_) + if verbose: printf(a) def link(repo_,arch_,file_): - """ Makes a link in the repo for the package """ - cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ - a=commands.getoutput(cmd_) - if verbose: - printf(cmd_ + a) + """ Makes a link in the repo for the package """ + cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ + a=commands.getoutput(cmd_) + if verbose: + printf(cmd_ + a) def add_free_repo(verbose_=verbose): - for repo_ in repo_list: - for arch_ in arch_list: - lista_=list() - for file_ in glob(free_path + repo_ + "/os/" + arch_ + "/*.pkg.tar.*"): - lista_.append(file_) - link(repo_,arch_,file_) - for dir_ in other: - for file_ in glob(free_path + repo_ + "/os/" + dir_ + "/*.pkg.tar.*"): - lista_.append(file_) - link(repo_,arch_,file_) - if lista_: - lista_=" ".join(lista_) - if verbose: printf(lista_) - cmd_="repo-add " + repodir + db(repo_,arch_) + " " + lista_ - printf(cmd_) - a=commands.getoutput(cmd_) - if verbose: printf(a) + cmd_=home + "/usr/bin/sync-free" + printf(cmd_) + a=commands.getoutput(cmd_) + if verbose_: printf(a) + for repo_ in repo_list: + for arch_ in arch_list: + lista_=list() + for file_ in glob(freedir + repo_ + "/os/" + arch_ + "/*.pkg.tar.*"): + lista_.append(file_) + # link(repo_,arch_,file_) + for dir_ in other: + for file_ in glob(freedir + repo_ + "/os/" + dir_ + "/*.pkg.tar.*"): + lista_.append(file_) + # link(repo_,arch_,file_) + + printf(lista_) + + if lista_: + lista_=" ".join(lista_) + if verbose: printf(lista_) + cmd_="repo-add " + repodir + db(repo_,arch_) + " " + lista_ + printf(cmd_) + a=commands.getoutput(cmd_) + if verbose: printf(a) def get_licenses(verbose_=verbose): - """ Extract the license from packages in repo_,arch_ and in pending_ file""" - cmd_=home + "/usr/bin/get_license.sh" - printf(cmd_) - a=commands.getoutput(cmd_) - if verbose_: printf(a) + """ Extract the license from packages in repo_,arch_ and in pending_ file""" + cmd_=home + "/usr/bin/get_license.sh" + printf(cmd_) + a=commands.getoutput(cmd_) + if verbose_: printf(a) def generate_rsync_command(base_command, dir_list=(repo_list + dir_list), destdir=repodir, source=mirror+mirrorpath, blacklist_file=False): @@ -207,33 +214,33 @@ def run_rsync(command,debug=verbose): return commands.getoutput(command) if __name__ == "__main__": - from time import time - start_time = time() - def minute(): - return str(round((time() - start_time)/60, 1)) - - printf(" Cleaning %s folder " % (tmp) ) - commands.getoutput("rm -r %s/*" % tmp) - printf(" Syncing repo") - sync_all_repo(True) - - printf(" Updating databases and pending files lists: minute %s \n" % minute() ) - for repo in repo_list: - for arch in arch_list: - printf( "\n" + repo + "-" + arch + "\n" ) - printf( "Get info: minute %s " % minute() ) - info=get_info(repo,arch) - printf( "Make pending: minute %s" % minute() ) - make_pending(repo,arch,info) - printf( "Update DB: minute %s" % minute() ) - remove_from_blacklist( - repo, arch, info, tuple( listado(blacklist) + listado(pending + "-" + repo + ".txt") ) ) - - printf("Adding Parabola Packages: minute %s\n" % minute() ) - add_free_repo(True) - - printf("Extracting licenses in pending: minute %s" % minute() ) - get_licenses() - - printf("\n\nDelay: %s minutes \n" % minute()) - + from time import time + start_time = time() + def minute(): + return str(round((time() - start_time)/60, 1)) + + printf(" Cleaning %s folder " % (tmp) ) + commands.getoutput("rm -r %s/*" % tmp) + printf(" Syncing repo") + sync_all_repo(True) + + printf(" Updating databases and pending files lists: minute %s \n" % minute() ) + for repo in repo_list: + for arch in arch_list: + printf( "\n" + repo + "-" + arch + "\n" ) + printf( "Get info: minute %s " % minute() ) + info=get_info(repo,arch) + printf( "Make pending: minute %s" % minute() ) + make_pending(repo,arch,info) + printf( "Update DB: minute %s" % minute() ) + remove_from_blacklist( + repo, arch, info, tuple( listado(blacklist) + listado(pending + "-" + repo + ".txt") ) ) + + printf("Adding Parabola Packages: minute %s\n" % minute() ) + add_free_repo(True) + + printf("Extracting licenses in pending: minute %s" % minute() ) + get_licenses() + + printf("\n\nDelay: %s minutes \n" % minute()) + -- cgit v1.2.3-2-g168b From df9a29ad9051f89658874959ebbc2f538e565771 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 16 Feb 2011 19:31:30 -0300 Subject: Applied changes for Parabola --- config | 22 +++++++++++----------- db-functions | 33 ++++++++++++++++++++++++--------- db-update | 9 +++------ 3 files changed, 38 insertions(+), 26 deletions(-) diff --git a/config b/config index 53191e0..e55958d 100644 --- a/config +++ b/config @@ -1,24 +1,24 @@ -FTP_BASE="/srv/ftp" -SVNREPO='' -PKGREPOS=() -PKGPOOL='' -SRCPOOL='' +FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" +SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" +PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing') +PKGPOOL='pool/packages' +SRCPOOL='sources/packages' -CLEANUP_DESTDIR="/srv/package-cleanup" +CLEANUP_DESTDIR="$FTP_BASE/old/packages" CLEANUP_DRYRUN=false # Time in days to keep moved packages CLEANUP_KEEP=30 -SOURCE_CLEANUP_DESTDIR="/srv/source-cleanup" +SOURCE_CLEANUP_DESTDIR="$FTP_BASE/old/sources" SOURCE_CLEANUP_DRYRUN=false # Time in days to keep moved sourcepackages -SOURCE_CLEANUP_KEEP=14 +SOURCE_CLEANUP_KEEP=30 LOCK_DELAY=10 LOCK_TIMEOUT=300 -STAGING="$HOME/staging" -TMPDIR="/srv/tmp" +STAGING="$FTP_BASE/staging" +TMPDIR="$HOME/tmp" ARCHES=(i686 x86_64) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" @@ -29,4 +29,4 @@ SRCEXT=".src.tar.gz" ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') # Override default config with config.local -[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" +#[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" diff --git a/db-functions b/db-functions index 7d431fc..0553188 100644 --- a/db-functions +++ b/db-functions @@ -74,7 +74,7 @@ in_array() { script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.$(basename $0)" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - local _owner="$(/usr/bin/stat -c %U $LOCKDIR)" + local _owner="$(stat -c %U $LOCKDIR)" error "Script $(basename $0) is already locked by $_owner." exit 1 else @@ -160,7 +160,7 @@ repo_lock () { _count=0 while [ $_count -le $_trial ] || $_lockblock ; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - _owner="$(/usr/bin/stat -c %U $LOCKDIR)" + _owner="$(stat -c %U $LOCKDIR)" warning "Repo [${1}] (${2}) is already locked by $_owner. " msg2 "Retrying in $LOCK_DELAY seconds..." else @@ -193,7 +193,7 @@ repo_unlock () { #repo_unlock _grep_pkginfo() { local _ret - _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 "^${2} = ")" + _ret="$(bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 "^${2} = ")" echo "${_ret#${2} = }" } @@ -358,9 +358,24 @@ check_splitpkgs() { if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" - svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ - "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null - [ $? -ge 1 ] && return 1 + +# Decide whether to look for PKGBUILD on [libre] or [libre-testing] + case $repo in + testing) + altrepo=libre-testing + ;; + *) + altrepo=libre + ;; + esac + + cp -r ${SVNREPO}/$repo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ + cp -r ${SVNREPO}/$altrepo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/$_pkgbase">/dev/null 2>&1 + + [[ $? -ge 1 ]] && { + echo "Failed $_pkgbase-$_pkgver-$_pkgarch" + return 1 + } fi local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) @@ -443,7 +458,7 @@ set_repo_permission() { local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" if [ -w "${dbfile}" ]; then - local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") + local group=$(stat --printf='%G' "$(dirname "${dbfile}")") chgrp $group "${dbfile}" || error "Could not change group of ${dbfile} to $group" chmod g+w "${dbfile}" || error "Could not set write permission for group $group to ${dbfile}" else @@ -458,7 +473,7 @@ arch_repo_add() { # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null - /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ + repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" @@ -474,7 +489,7 @@ arch_repo_remove() { error "No database found at '${dbfile}'" return 1 fi - /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ + repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ || error "repo-remove ${dbfile} ${pkgs[@]}" set_repo_permission "${repo}" "${arch}" } diff --git a/db-update b/db-update index 5bdce5e..4740809 100755 --- a/db-update +++ b/db-update @@ -35,12 +35,9 @@ for repo in ${repos[@]}; do if ! check_pkgfile "${pkg}"; then die "Package ${repo}/$(basename ${pkg}) is not consistent with its meta data" fi - if ! check_pkgsvn "${pkg}" "${repo}"; then - die "Package ${repo}/$(basename ${pkg}) is not consistent with svn repository" - fi - if ! check_pkgrepos "${pkg}"; then - die "Package ${repo}/$(basename ${pkg}) already exists in another repository" - fi + #if ! check_pkgrepos "${pkg}"; then + # die "Package ${repo}/$(basename ${pkg}) already exists in another repository" + #fi done if ! check_splitpkgs ${repo} ${pkgs[@]}; then die "Missing split packages for ${repo}" -- cgit v1.2.3-2-g168b From 11db46275d14c9a9c2e59019fed3de4b8803f9f4 Mon Sep 17 00:00:00 2001 From: Parabola Date: Fri, 18 Feb 2011 15:42:59 -0800 Subject: Parabola specific changes --- config | 24 ++++++++++++------------ db-functions | 33 ++++++++++++++++++++++++--------- db-update | 9 +++------ 3 files changed, 39 insertions(+), 27 deletions(-) diff --git a/config b/config index 53191e0..720d110 100644 --- a/config +++ b/config @@ -1,25 +1,25 @@ -FTP_BASE="/srv/ftp" -SVNREPO='' -PKGREPOS=() -PKGPOOL='' -SRCPOOL='' +FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" +SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" +PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing') +PKGPOOL='pool/packages' +SRCPOOL='sources/packages' -CLEANUP_DESTDIR="/srv/package-cleanup" +CLEANUP_DESTDIR="$FTP_BASE/old/packages" CLEANUP_DRYRUN=false # Time in days to keep moved packages CLEANUP_KEEP=30 -SOURCE_CLEANUP_DESTDIR="/srv/source-cleanup" +SOURCE_CLEANUP_DESTDIR="$FTP_BASE/old/sources" SOURCE_CLEANUP_DRYRUN=false # Time in days to keep moved sourcepackages -SOURCE_CLEANUP_KEEP=14 +SOURCE_CLEANUP_KEEP=30 LOCK_DELAY=10 LOCK_TIMEOUT=300 -STAGING="$HOME/staging" -TMPDIR="/srv/tmp" -ARCHES=(i686 x86_64) +STAGING="$FTP_BASE/staging" +TMPDIR="$HOME/tmp" +ARCHES=(i686 x86_64 mipsel) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.*" @@ -29,4 +29,4 @@ SRCEXT=".src.tar.gz" ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') # Override default config with config.local -[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" +#[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" diff --git a/db-functions b/db-functions index 7d431fc..0553188 100644 --- a/db-functions +++ b/db-functions @@ -74,7 +74,7 @@ in_array() { script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.$(basename $0)" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - local _owner="$(/usr/bin/stat -c %U $LOCKDIR)" + local _owner="$(stat -c %U $LOCKDIR)" error "Script $(basename $0) is already locked by $_owner." exit 1 else @@ -160,7 +160,7 @@ repo_lock () { _count=0 while [ $_count -le $_trial ] || $_lockblock ; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - _owner="$(/usr/bin/stat -c %U $LOCKDIR)" + _owner="$(stat -c %U $LOCKDIR)" warning "Repo [${1}] (${2}) is already locked by $_owner. " msg2 "Retrying in $LOCK_DELAY seconds..." else @@ -193,7 +193,7 @@ repo_unlock () { #repo_unlock _grep_pkginfo() { local _ret - _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 "^${2} = ")" + _ret="$(bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 "^${2} = ")" echo "${_ret#${2} = }" } @@ -358,9 +358,24 @@ check_splitpkgs() { if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" - svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ - "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null - [ $? -ge 1 ] && return 1 + +# Decide whether to look for PKGBUILD on [libre] or [libre-testing] + case $repo in + testing) + altrepo=libre-testing + ;; + *) + altrepo=libre + ;; + esac + + cp -r ${SVNREPO}/$repo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ + cp -r ${SVNREPO}/$altrepo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/$_pkgbase">/dev/null 2>&1 + + [[ $? -ge 1 ]] && { + echo "Failed $_pkgbase-$_pkgver-$_pkgarch" + return 1 + } fi local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) @@ -443,7 +458,7 @@ set_repo_permission() { local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" if [ -w "${dbfile}" ]; then - local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") + local group=$(stat --printf='%G' "$(dirname "${dbfile}")") chgrp $group "${dbfile}" || error "Could not change group of ${dbfile} to $group" chmod g+w "${dbfile}" || error "Could not set write permission for group $group to ${dbfile}" else @@ -458,7 +473,7 @@ arch_repo_add() { # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null - /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ + repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" @@ -474,7 +489,7 @@ arch_repo_remove() { error "No database found at '${dbfile}'" return 1 fi - /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ + repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ || error "repo-remove ${dbfile} ${pkgs[@]}" set_repo_permission "${repo}" "${arch}" } diff --git a/db-update b/db-update index 5bdce5e..4740809 100755 --- a/db-update +++ b/db-update @@ -35,12 +35,9 @@ for repo in ${repos[@]}; do if ! check_pkgfile "${pkg}"; then die "Package ${repo}/$(basename ${pkg}) is not consistent with its meta data" fi - if ! check_pkgsvn "${pkg}" "${repo}"; then - die "Package ${repo}/$(basename ${pkg}) is not consistent with svn repository" - fi - if ! check_pkgrepos "${pkg}"; then - die "Package ${repo}/$(basename ${pkg}) already exists in another repository" - fi + #if ! check_pkgrepos "${pkg}"; then + # die "Package ${repo}/$(basename ${pkg}) already exists in another repository" + #fi done if ! check_splitpkgs ${repo} ${pkgs[@]}; then die "Missing split packages for ${repo}" -- cgit v1.2.3-2-g168b From 439bd2112c01bc9b185d2552aa0888d7055929e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Fri, 18 Feb 2011 20:43:56 -0300 Subject: First stab at sourceballing everything --- config | 2 +- config.local.gerolde | 4 ---- config.local.sigurd | 4 ---- cron-jobs/sourceballs | 13 ++++++++----- 4 files changed, 9 insertions(+), 14 deletions(-) delete mode 100644 config.local.gerolde delete mode 100644 config.local.sigurd diff --git a/config b/config index e55958d..720d110 100644 --- a/config +++ b/config @@ -19,7 +19,7 @@ LOCK_TIMEOUT=300 STAGING="$FTP_BASE/staging" TMPDIR="$HOME/tmp" -ARCHES=(i686 x86_64) +ARCHES=(i686 x86_64 mipsel) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.*" diff --git a/config.local.gerolde b/config.local.gerolde deleted file mode 100644 index 4501a93..0000000 --- a/config.local.gerolde +++ /dev/null @@ -1,4 +0,0 @@ -PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable') -PKGPOOL='pool/packages' -SRCPOOL='sources/packages' -SVNREPO='file:///srv/svn-packages' diff --git a/config.local.sigurd b/config.local.sigurd deleted file mode 100644 index d28aa37..0000000 --- a/config.local.sigurd +++ /dev/null @@ -1,4 +0,0 @@ -PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing') -PKGPOOL='pool/community' -SRCPOOL='sources/community' -SVNREPO='file:///srv/svn-packages' diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index b55de05..0df007e 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -63,10 +63,11 @@ for repo in ${PKGREPOS[@]}; do if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then continue fi + # Commenting out, we'll sourceball everything # Check if the license or .force file does not enforce creating a source package - if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then - continue - fi +# if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then +# continue +# fi # Store the expected file name of the source package echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" @@ -79,8 +80,10 @@ for repo in ${PKGREPOS[@]}; do # Get the sources from svn mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" - svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ - "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 + #svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ + # "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 + cp -r "${SVNREPO}/libre/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ + cp -r "${SVNREPO}/libre-testing/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 if [ $? -ge 1 ]; then failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" continue -- cgit v1.2.3-2-g168b From f8c49cc2915f66b53bd9dd248b41ab6a96c1059a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sun, 20 Feb 2011 22:09:28 -0300 Subject: Changes to sourceball everything --- config | 1 + cron-jobs/sourceballs | 15 +++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/config b/config index 720d110..12bdaaa 100644 --- a/config +++ b/config @@ -1,4 +1,5 @@ FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" +ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing') PKGPOOL='pool/packages' diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 0df007e..5726484 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -21,10 +21,10 @@ renice +10 -p $$ > /dev/null for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do # Repo does not exist; skip it - if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + if [ ! -f "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue fi - bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ + bsdtar -xOf "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ | awk '/^%NAME%/ { getline b }; /^%BASE%/ { getline b }; /^%VERSION%/ { getline v }; @@ -46,7 +46,7 @@ for repo in ${PKGREPOS[@]}; do done # Create a list of all available source package file names -find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" +find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" # Check for all packages if we need to build a source package for repo in ${PKGREPOS[@]}; do @@ -59,7 +59,7 @@ for repo in ${PKGREPOS[@]}; do pkgarch=${pkginfo[2]} pkglicense=(${pkginfo[@]:3}) - # Should this package be skipped? + # Should this packages be skipped? if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then continue fi @@ -82,6 +82,9 @@ for repo in ${PKGREPOS[@]}; do mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" #svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ # "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 + + # If it's on official repos, nor [libre], nor [libre-testing] + cp -r "${SVNREPO}/$repo/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ cp -r "${SVNREPO}/libre/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ cp -r "${SVNREPO}/libre-testing/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 if [ $? -ge 1 ]; then @@ -93,7 +96,7 @@ for repo in ${PKGREPOS[@]}; do pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null makepkg --nocolor --allsource --ignorearch >/dev/null 2>&1 if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then - mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" + mv "${pkgbase}-${pkgver}${SRCEXT}" "${ARCH_BASE}/${SRCPOOL}" # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" newpkgs[${#newpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" @@ -129,7 +132,7 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" if ! ${SOURCE_CLEANUP_DRYRUN}; then - mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" fi done -- cgit v1.2.3-2-g168b From 0162a16595059e269ba2d7b47e69212bc2ab1336 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sun, 20 Feb 2011 22:09:40 -0300 Subject: Changed looking for a package on abslibre logic --- db-functions | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/db-functions b/db-functions index 0553188..e2731cb 100644 --- a/db-functions +++ b/db-functions @@ -1,5 +1,7 @@ #!/bin/bash +. config + # Some PKGBUILDs need CARCH to be set CARCH="x86_64" @@ -16,7 +18,7 @@ restore_umask () { } # set up general environment -WORKDIR=$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX) +WORKDIR=$(mktemp -d ${TMPDIR}/$(basename $0).XXXXXXXXXX) LOCKS=() # check if messages are to be printed using color @@ -359,18 +361,9 @@ check_splitpkgs() { if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" -# Decide whether to look for PKGBUILD on [libre] or [libre-testing] - case $repo in - testing) - altrepo=libre-testing - ;; - *) - altrepo=libre - ;; - esac - cp -r ${SVNREPO}/$repo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ - cp -r ${SVNREPO}/$altrepo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/$_pkgbase">/dev/null 2>&1 + cp -r ${SVNREPO}/libre/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ + cp -r ${SVNREPO}/libre-testing/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/$_pkgbase">/dev/null 2>&1 [[ $? -ge 1 ]] && { echo "Failed $_pkgbase-$_pkgver-$_pkgarch" -- cgit v1.2.3-2-g168b From c6516eca2803b780a01a6b2bf3e081c8bd570301 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 20 Feb 2011 17:10:40 -0800 Subject: sourceballs --- config | 1 + cron-jobs/sourceballs | 12 ++++++------ db-functions | 5 ++++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/config b/config index 720d110..12bdaaa 100644 --- a/config +++ b/config @@ -1,4 +1,5 @@ FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" +ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing') PKGPOOL='pool/packages' diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 0df007e..53d43ce 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -21,10 +21,10 @@ renice +10 -p $$ > /dev/null for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do # Repo does not exist; skip it - if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + if [ ! -f "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue fi - bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ + bsdtar -xOf "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ | awk '/^%NAME%/ { getline b }; /^%BASE%/ { getline b }; /^%VERSION%/ { getline v }; @@ -46,7 +46,7 @@ for repo in ${PKGREPOS[@]}; do done # Create a list of all available source package file names -find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" +find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" # Check for all packages if we need to build a source package for repo in ${PKGREPOS[@]}; do @@ -91,9 +91,9 @@ for repo in ${PKGREPOS[@]}; do # Build the actual source package pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null - makepkg --nocolor --allsource --ignorearch >/dev/null 2>&1 + makepkg --nocolor --allsource --ignorearch # >/dev/null 2>&1 if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then - mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" + mv "${pkgbase}-${pkgver}${SRCEXT}" "${ARCH_BASE}/${SRCPOOL}" # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" newpkgs[${#newpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" @@ -129,7 +129,7 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" if ! ${SOURCE_CLEANUP_DRYRUN}; then - mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" fi done diff --git a/db-functions b/db-functions index 0553188..e790fb6 100644 --- a/db-functions +++ b/db-functions @@ -16,7 +16,7 @@ restore_umask () { } # set up general environment -WORKDIR=$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX) +WORKDIR=$(mktemp -d /home/parabolavnx/tmp/$(basename $0).XXXXXXXXXX) LOCKS=() # check if messages are to be printed using color @@ -376,6 +376,9 @@ check_splitpkgs() { echo "Failed $_pkgbase-$_pkgver-$_pkgarch" return 1 } + + sleep 20s + fi local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) -- cgit v1.2.3-2-g168b From 9793aaf4e9a716f126e71a1db412cbd32f161593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Thu, 3 Mar 2011 20:11:33 -0600 Subject: * repo-maintainer doesn't take out reading permission from pending files * Removed commented code --- pato2.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pato2.py b/pato2.py index f5deb48..21dde16 100644 --- a/pato2.py +++ b/pato2.py @@ -128,12 +128,6 @@ def remove_from_blacklist(repo_,arch_,info_,blacklist_): printf(com_) a = commands.getoutput(com_) if verbose: printf(a) - if pack_: - pack_=" ".join(pack_) - com_="chmod a-r " + pack_ - printf(com_) - a=commands.getoutput(com_) - if verbose: printf(a) def link(repo_,arch_,file_): """ Makes a link in the repo for the package """ @@ -152,11 +146,9 @@ def add_free_repo(verbose_=verbose): lista_=list() for file_ in glob(freedir + repo_ + "/os/" + arch_ + "/*.pkg.tar.*"): lista_.append(file_) - # link(repo_,arch_,file_) for dir_ in other: for file_ in glob(freedir + repo_ + "/os/" + dir_ + "/*.pkg.tar.*"): lista_.append(file_) - # link(repo_,arch_,file_) printf(lista_) -- cgit v1.2.3-2-g168b From cf2baeb05af80a849bfc2192f3af1e741e50745d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sat, 5 Mar 2011 01:25:40 -0300 Subject: sourceballs2 creates sourceballs from the entire ABSLibre --- config | 6 +--- cron-jobs/sourceballs2 | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 5 deletions(-) create mode 100644 cron-jobs/sourceballs2 diff --git a/config b/config index 12bdaaa..f08ad2d 100644 --- a/config +++ b/config @@ -26,8 +26,4 @@ FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.*" SRCEXT=".src.tar.gz" -# Allowed licenses: get sourceballs only for licenses in this array -ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') - -# Override default config with config.local -#[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" +MAKEPKGCONF="$HOME/etc/makepkg.conf" diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 new file mode 100644 index 0000000..ba35298 --- /dev/null +++ b/cron-jobs/sourceballs2 @@ -0,0 +1,81 @@ +#!/bin/bash +# Steps +# Traverse ABSLibre +# Makepkg --allsource every package +# Remove the old sourceballs + +dirname="$(dirname $(readlink -e $0))" +. "${dirname}/../db-functions" +. "${dirname}/../config" +. "${MAKEPKGCONF}" + +pushd "${WORKDIR}" >/dev/null + +script_lock + +#adjust the nice level to run at a lower priority +renice +10 -p $$ > /dev/null + +# Create a list of all available source package file names +find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" + +pushd "${SVNREPO}" >/dev/null + +failedpkgs=() +for repo in ${PKGREPOS[@]}; do + pushd $repo >/dev/null + find . -maxdepth 1 -type d | while read pkg; do + pushd "${SVNREPO}/$repo/$pkg" >/dev/null + + [[ ! -e PKGBUILD ]] && { + warning "$repo/$pkg is not a package" + continue + } + + unset pkgbase pkgname + source PKGBUILD + pkgbase=${pkgbase:-$pkgname} + + echo "${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" + + # Skip already sourceballed + [[ -e "${SRCPKGDEST}/${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" ]] && \ + continue + + makepkg --allsource --ignorearch -c + + [[ $? -ne 0 ]] && \ + failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" + + done + popd >/dev/null +done + +# Cleanup old source packages +cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" +cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort" +old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) + +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages..." + ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + if ! ${SOURCE_CLEANUP_DRYRUN}; then + mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + fi + done +fi + +old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -name "*${SRCEXT}" -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages from the cleanup directory..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + ${SOURCE_CLEANUP_DRYRUN} || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + done +fi + +script_unlock + -- cgit v1.2.3-2-g168b From a3694fd2824f6aab2905f52f64a138437f7e004b Mon Sep 17 00:00:00 2001 From: Parabola Date: Sat, 5 Mar 2011 10:56:49 -0800 Subject: =?UTF-8?q?Taken=20out=20=C2=ABsources=C2=BB=20from=20dir=5Flist?= =?UTF-8?q?=20for=20rsync?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/config.py b/config.py index 6420355..b198450 100644 --- a/config.py +++ b/config.py @@ -27,9 +27,10 @@ archdb = tmp + "/db" free_path= path + "/free/" # Repo, arch, and other folders to use for repo -repo_list = ("core", "extra", "community", "testing", "community-testing", "multilib") -dir_list = ("pool","sources") -arch_list = ("i686", "x86_64") +# This are tuples, so **always keep a comma before closing parenthesis ** +repo_list = ("core", "extra", "community", "testing", "community-testing", "multilib",) +dir_list = ("pool",) +arch_list = ("i686", "x86_64",) other = ("any",) # Output -- cgit v1.2.3-2-g168b From 5a4480f1f3e2e7d0aec965b21f72f4454618333e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sat, 5 Mar 2011 13:13:41 -0600 Subject: =?UTF-8?q?*=20Added=20=C2=ABdepends=C2=BB=20field=20to=20Package?= =?UTF-8?q?=20class.=20*=20Merged=20tests=20for=20filter=20functions=20in?= =?UTF-8?q?=20test=5Ffilter.py=20*=20Added=20testfiles=20for=20test=5Ffilt?= =?UTF-8?q?er.py=20in=20test=20directory=20*=20Added=20pkginfo=5Ffrom=5Fde?= =?UTF-8?q?sc=20function=20to=20filter.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.py | 3 +- filter.py | 35 ++++++- test/core.db.tar.gz | Bin 0 -> 1637 bytes test/depends | 4 + test/desc | 39 ++++++++ test/test_filter.py | 169 +++++++++++++++++++++++++++++++++ test/test_pkginfo_from_file.py | 76 --------------- test/test_pkginfo_from_rsync_output.py | 65 ------------- 8 files changed, 248 insertions(+), 143 deletions(-) create mode 100644 test/core.db.tar.gz create mode 100644 test/depends create mode 100644 test/desc create mode 100644 test/test_filter.py delete mode 100644 test/test_pkginfo_from_file.py delete mode 100644 test/test_pkginfo_from_rsync_output.py diff --git a/config.py b/config.py index 6420355..688d6b5 100644 --- a/config.py +++ b/config.py @@ -62,7 +62,8 @@ class Package: "release" : False, "arch" : False, "license" : False, - "location": False} + "location": False, + "depends" : False,} def __setitem__(self, key, item): if key in self.package_info.keys(): diff --git a/filter.py b/filter.py index f91ec68..2fe61f0 100644 --- a/filter.py +++ b/filter.py @@ -26,6 +26,37 @@ def pkginfo_from_filename(filename): pkg["name"] = "-".join(fileattrs) return pkg +def pkginfo_from_desc(filename): + """ Returns pkginfo from desc file. + + Parameters: + ---------- + filename -> str File must exist + + Returns: + ---------- + pkg -> Package object""" + if not os.path.isfile(filename): + raise NonValidFile + try: + f=open(filename) + info=f.read().rsplit() + finally: + f.close() + pkg = Package() + info_map={"name" :("%NAME%" , None), + "version" :("%VERSION%" , 0 ), + "release" :("%VERSION%" , 1 ), + "arch" :("%ARCH%" , None), + "license" :("%LICENSE%" , None), + "location":("%FILENAME%", None),} + + for key in info_map.keys(): + field,pos=info_map[key] + pkg[key]=info[info.index(field)+1] + if pos is not None: + pkg[key]=pkg[key].split("-")[pos] + return pkg def pkginfo_from_rsync_output(rsync_output): """ Generates a list of packages and versions from an rsync output @@ -45,7 +76,7 @@ def pkginfo_from_rsync_output(rsync_output): return pkginfo_from_filename(line.rsplit()[location_field]) def do_nothing(): - """""" + pass options = { "d": do_nothing, "l": package_or_link, @@ -84,6 +115,8 @@ def pkginfo_from_files_in_dir(directory): package_list.append(pkginfo_from_filename(filename)) return tuple(package_list) + + def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, exclude_file=rsync_blacklist, debug=verbose): """ Generate an exclude list for rsync diff --git a/test/core.db.tar.gz b/test/core.db.tar.gz new file mode 100644 index 0000000..a28ea64 Binary files /dev/null and b/test/core.db.tar.gz differ diff --git a/test/depends b/test/depends new file mode 100644 index 0000000..7ff3ad4 --- /dev/null +++ b/test/depends @@ -0,0 +1,4 @@ +%DEPENDS% +glibc>=2.13 +zlib + diff --git a/test/desc b/test/desc new file mode 100644 index 0000000..abba644 --- /dev/null +++ b/test/desc @@ -0,0 +1,39 @@ +%FILENAME% +binutils-2.21-4-x86_64.pkg.tar.xz + +%NAME% +binutils + +%VERSION% +2.21-4 + +%DESC% +A set of programs to assemble and manipulate binary and object files + +%GROUPS% +base + +%CSIZE% +3412892 + +%ISIZE% +17571840 + +%MD5SUM% +4e666f87c78998f4839f33dc06d2043a + +%URL% +http://www.gnu.org/software/binutils/ + +%LICENSE% +GPL + +%ARCH% +x86_64 + +%BUILDDATE% +1297240369 + +%PACKAGER% +Allan McRae + diff --git a/test/test_filter.py b/test/test_filter.py new file mode 100644 index 0000000..5127b75 --- /dev/null +++ b/test/test_filter.py @@ -0,0 +1,169 @@ +# -*- encoding: utf-8 -*- +""" """ + +__author__ = "Joshua Ismael Haase Hernández " +__version__ = "$Revision: 1.1 $" +__date__ = "$Date: 2011/02/08 $" +__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" +__license__ = "GPL3+" + +from repm.config import * +from repm.filter import * +import unittest + +class pkginfo_from_file_KnownValues(unittest.TestCase): + # (filename, name, version, release, arch) + # filename is location + known=( + ("community-testing/os/i686/inputattach-1.24-3-i686.pkg.tar.xz","inputattach","1.24","3","i686"), + ("community-testing/os/i686/ngspice-22-1-i686.pkg.tar.xz","ngspice","22","1","i686"), + ("community-testing/os/i686/tmux-1.4-2-i686.pkg.tar.xz","tmux","1.4","2","i686"), + ("community-testing/os/i686/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), + ("../../../pool/community/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), + ("community-testing/os/x86_64/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), + ("../../../pool/community/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), + ("tor-0.2.1.29-2-x86_64.pkg.tar.xz","tor","0.2.1.29","2","x86_64"), + ) + + def generate_results(self, example_tuple, attr): + location, name, version, release, arch = example_tuple + return pkginfo_from_filename(location)[attr], locals()[attr] + + def testReturnPackageObject(self): + for i in self.known: + location, name, version, release, arch = i + self.assertIsInstance(pkginfo_from_filename(location),Package) + + def testNames(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="name") + self.assertEqual(k, v) + + def testVersions(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="version") + self.assertEqual(k, v) + + def testArchs(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="arch") + self.assertEqual(k, v) + + def testReleases(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="release") + self.assertEqual(k, v) + + def testLocations(self): + for i in self.known: + k,v = self.generate_results(example_tuple=i,attr="location") + self.assertEqual(k, v) + +class pkginfo_from_file_BadInput(unittest.TestCase): + bad=("community-testing/os/i686/community-testing.db", + "community-testing/os/i686/community-testing.db.tar.gz", + "community-testing/os/i686/community-testing.db.tar.gz.old", + "community-testing/os/i686/community-testing.files", + "community-testing/os/i686/community-testing.files.tar.gz", + "community-testing/os/x86_64") + + def testBadInput(self): + for i in self.bad: + self.assertRaises(NonValidFile,pkginfo_from_filename,i) + +class pkginfoFromRsyncOutput(unittest.TestCase): + example_package_list=(Package(),Package(),Package()) + example_package_list[0].package_info={ "name" : "alex", + "version" : "2.3.4", + "release" : "1", + "arch" : "i686", + "license" : False, + "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz", + "depends" : False,} + example_package_list[1].package_info={ "name" : "any2dvd", + "version" : "0.34", + "release" : "4", + "arch" : "any", + "license" : False, + "location": "community/os/any/any2dvd-0.34-4-any.pkg.tar.xz", + "depends" : False,} + example_package_list[2].package_info={ "name" : "gmime22", + "version" : "2.2.26", + "release" : "1", + "arch" : "x86_64", + "license" : False, + "location": "community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz", + "depends" : False,} + + try: + output_file = open("rsync_output_sample") + rsync_out= output_file.read() + output_file.close() + except IOError: print("There is no rsync_output_sample file") + + pkglist = pkginfo_from_rsync_output(rsync_out) + + def testOutputArePackages(self): + if not self.pkglist: + self.fail("not pkglist:" + str(self.pkglist)) + for pkg in self.pkglist: + self.assertIsInstance(pkg,Package) + + def testPackageInfo(self): + if not self.pkglist: + self.fail("Pkglist doesn't exist: " + str(self.pkglist)) + self.assertEqual(self.pkglist,self.example_package_list) + +class generateRsyncBlacklist(unittest.TestCase): + example_package_list=(Package(),Package(),Package()) + example_package_list[0].package_info={ "name" : "alex", + "version" : "2.3.4", + "release" : "1", + "arch" : "i686", + "license" : False, + "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz", + "depends" : False,} + example_package_list[1].package_info={ "name" : "any2dvd", + "version" : "0.34", + "release" : "4", + "arch" : "any", + "license" : False, + "location": "community/os/any/any2dvd-0.34-4-any.pkg.tar.xz", + "depends" : False,} + example_package_list[2].package_info={ "name" : "gmime22", + "version" : "2.2.26", + "release" : "1", + "arch" : "x86_64", + "license" : False, + "location": "community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz", + "depends" : False,} + + def testListado(self): + self.assertEqual(listado("blacklist_sample"),["alex","gmime22"]) + + def testExcludeFiles(self): + a=generate_exclude_list_from_blacklist(self.example_package_list,listado("blacklist_sample"),debug=True) + b=[self.example_package_list[0]["location"],self.example_package_list[2]["location"]] + self.assertEqual(a,b) + +class pkginfo_from_descKnownValues(unittest.TestCase): + pkgsample=Package() + pkgsample.package_info={"name" : "binutils", + "version" : "2.21", + "release" : "4", + "arch" : "x86_64", + "license" : "GPL", + "location": "binutils-2.21-4-x86_64.pkg.tar.xz", + "depends" : False,} + pkggen=pkginfo_from_desc("desc") + def testPkginfoFromDesc(self): + if self.pkggen is None: + self.fail("return value is None") + self.assertEqual(self.pkgsample,self.pkggen) + +class pkginfo_from_db(unittest.TestCase): + archdb = os.path.join("./workdir") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_pkginfo_from_file.py b/test/test_pkginfo_from_file.py deleted file mode 100644 index 0d7ede6..0000000 --- a/test/test_pkginfo_from_file.py +++ /dev/null @@ -1,76 +0,0 @@ -#! /usr/bin/python -# -*- encoding: utf-8 -*- -""" """ - -__author__ = "Joshua Ismael Haase Hernández " -__version__ = "$Revision: 1.1 $" -__date__ = "$Date: 2011/02/08 $" -__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" -__license__ = "GPL3+" - -from repm.config import * -from repm.filter import * -import unittest - -class KnownValues(unittest.TestCase): - # (filename, name, version, release, arch) - # filename is location - known=( - ("community-testing/os/i686/inputattach-1.24-3-i686.pkg.tar.xz","inputattach","1.24","3","i686"), - ("community-testing/os/i686/ngspice-22-1-i686.pkg.tar.xz","ngspice","22","1","i686"), - ("community-testing/os/i686/tmux-1.4-2-i686.pkg.tar.xz","tmux","1.4","2","i686"), - ("community-testing/os/i686/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), - ("../../../pool/community/tor-0.2.1.29-2-i686.pkg.tar.xz","tor","0.2.1.29","2","i686"), - ("community-testing/os/x86_64/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), - ("../../../pool/community/inputattach-1.24-3-x86_64.pkg.tar.xz","inputattach","1.24","3","x86_64"), - ("tor-0.2.1.29-2-x86_64.pkg.tar.xz","tor","0.2.1.29","2","x86_64"), - ) - - def generate_results(self, example_tuple, attr): - location, name, version, release, arch = example_tuple - return pkginfo_from_filename(location)[attr], locals()[attr] - - def testReturnPackageObject(self): - for i in self.known: - location, name, version, release, arch = i - self.assertIsInstance(pkginfo_from_filename(location),Package) - - def testNames(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="name") - self.assertEqual(k, v) - - def testVersions(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="version") - self.assertEqual(k, v) - - def testArchs(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="arch") - self.assertEqual(k, v) - - def testReleases(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="release") - self.assertEqual(k, v) - - def testLocations(self): - for i in self.known: - k,v = self.generate_results(example_tuple=i,attr="location") - self.assertEqual(k, v) - -class BadInput(unittest.TestCase): - bad=("community-testing/os/i686/community-testing.db", - "community-testing/os/i686/community-testing.db.tar.gz", - "community-testing/os/i686/community-testing.db.tar.gz.old", - "community-testing/os/i686/community-testing.files", - "community-testing/os/i686/community-testing.files.tar.gz", - "community-testing/os/x86_64") - - def testBadInput(self): - for i in self.bad: - self.assertRaises(NonValidFile,pkginfo_from_filename,i) - -if __name__ == "__main__": - unittest.main() diff --git a/test/test_pkginfo_from_rsync_output.py b/test/test_pkginfo_from_rsync_output.py deleted file mode 100644 index aca49e8..0000000 --- a/test/test_pkginfo_from_rsync_output.py +++ /dev/null @@ -1,65 +0,0 @@ -# -*- encoding: utf-8 -*- -""" """ - -__author__ = "Joshua Ismael Haase Hernández " -__version__ = "$Revision: 1.1 $" -__date__ = "$Date: 2011/02/08 $" -__copyright__ = "Copyright (c) 2011 Joshua Ismael Haase Hernández" -__license__ = "GPL3+" - -from repm.config import * -from repm.filter import * -import unittest - -example_package_list=(Package(),Package(),Package()) -example_package_list[0].package_info={ "name" : "alex", - "version" : "2.3.4", - "release" : "1", - "arch" : "i686", - "license" : False, - "location": "community-staging/os/i686/alex-2.3.4-1-i686.pkg.tar.xz"} -example_package_list[1].package_info={ "name" : "any2dvd", - "version" : "0.34", - "release" : "4", - "arch" : "any", - "license" : False, - "location": "community/os/any/any2dvd-0.34-4-any.pkg.tar.xz"} -example_package_list[2].package_info={ "name" : "gmime22", - "version" : "2.2.26", - "release" : "1", - "arch" : "x86_64", - "license" : False, - "location": "community/os/x86_64/gmime22-2.2.26-1-x86_64.pkg.tar.xz"} - -class pkginfoFromRsyncOutput(unittest.TestCase): - try: - output_file = open("rsync_output_sample") - rsync_out= output_file.read() - output_file.close() - except IOError: print("There is no rsync_output_sample file") - - pkglist = pkginfo_from_rsync_output(rsync_out) - - def testOutputArePackages(self): - if not self.pkglist: - self.fail("not pkglist:" + str(self.pkglist)) - for pkg in self.pkglist: - self.assertIsInstance(pkg,Package) - - def testPackageInfo(self): - if not self.pkglist: - self.fail("Pkglist doesn't exist: " + str(self.pkglist)) - self.assertEqual(self.pkglist,example_package_list) - -class generateRsyncBlacklist(unittest.TestCase): - """ Test Blacklist generation """ - def testListado(self): - self.assertEqual(listado("blacklist_sample"),["alex","gmime22"]) - - def testExcludeFiles(self): - a=generate_exclude_list_from_blacklist(example_package_list,listado("blacklist_sample"),debug=True) - b=[example_package_list[0]["location"],example_package_list[2]["location"]] - self.assertEqual(a,b) - -if __name__ == "__main__": - unittest.main() -- cgit v1.2.3-2-g168b From 8c9ff0c97d0802fadc96fe9eacaef7eaa2db5ad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 6 Mar 2011 16:37:36 -0600 Subject: * Using os.path.join for joining path instead of str + str to avoid errors --- pato2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pato2.py b/pato2.py index 21dde16..936e97a 100644 --- a/pato2.py +++ b/pato2.py @@ -137,7 +137,7 @@ def link(repo_,arch_,file_): printf(cmd_ + a) def add_free_repo(verbose_=verbose): - cmd_=home + "/usr/bin/sync-free" + cmd_=os.path.join(home,"/usr/bin/sync-free") printf(cmd_) a=commands.getoutput(cmd_) if verbose_: printf(a) -- cgit v1.2.3-2-g168b From 31a0d9e077ed10c5a4b9f461bbde8979f127bdf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 6 Mar 2011 16:43:18 -0600 Subject: * Changed rsync-update-command to not delete (delete made by ftp-dir-cleanup) --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index ada17f0..37bebc0 100644 --- a/config.py +++ b/config.py @@ -46,7 +46,7 @@ rsync_blacklist = docs + "/rsyncBlacklist" # Rsync commands rsync_list_command="rsync -a --no-motd --list-only " -rsync_update_command="rsync -av --delete-after --delay-updates " +rsync_update_command="rsync -av --delay-updates " # Classes and Exceptions class NonValidFile(ValueError): pass -- cgit v1.2.3-2-g168b From 229a9c504cbd733c93cf91399dc54bedf5160cc5 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 6 Mar 2011 15:22:21 -0800 Subject: Fixes --- config | 8 ++--- cron-jobs/sourceballs2 | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++ db-functions | 4 +-- 3 files changed, 90 insertions(+), 9 deletions(-) create mode 100755 cron-jobs/sourceballs2 diff --git a/config b/config index 12bdaaa..217b627 100644 --- a/config +++ b/config @@ -20,14 +20,10 @@ LOCK_TIMEOUT=300 STAGING="$FTP_BASE/staging" TMPDIR="$HOME/tmp" -ARCHES=(i686 x86_64 mipsel) +ARCHES=(i686 x86_64 mips64el) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.*" SRCEXT=".src.tar.gz" -# Allowed licenses: get sourceballs only for licenses in this array -ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') - -# Override default config with config.local -#[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" +MAKEPKGCONF="$HOME/etc/makepkg.conf" diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 new file mode 100755 index 0000000..452208e --- /dev/null +++ b/cron-jobs/sourceballs2 @@ -0,0 +1,87 @@ +#!/bin/bash + +dirname="$(dirname $(readlink -e $0))" +. "${dirname}/../db-functions" +. "${dirname}/../config" +. "${MAKEPKGCONF}" + +pushd "${WORKDIR}" >/dev/null + +script_lock + +#adjust the nice level to run at a lower priority +renice +10 -p $$ > /dev/null + +# Create a list of all available source package file names +find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" + +# Steps +# Traverse the ABSLibre +# Makepkg --allsource every package +# Remove the old packages +pushd "${SVNREPO}" >/dev/null + +failedpkgs=() +for repo in ${PKGREPOS[@]}; do + pushd $repo >/dev/null + find . -maxdepth 1 -type d | while read pkg; do + pushd "${SVNREPO}/$repo/$pkg" >/dev/null + + [[ ! -e PKGBUILD ]] && { + warning "$repo/$pkg is not a package" + continue + } + + unset pkgbase pkgname + source PKGBUILD + pkgbase=${pkgbase:-$pkgname} + + echo "${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" + + # Skip already sourceballed + [[ -e "${SRCPKGDEST}/${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" ]] && \ + continue + + makepkg --allsource --ignorearch -c >/dev/null 2>&1 + + [[ $? -ne 0 ]] && \ + failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" + + done + popd >/dev/null +done + +# Cleanup old source packages +cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" +cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort" +old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) + +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages..." + ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + if ! ${SOURCE_CLEANUP_DRYRUN}; then + mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + fi + done +fi + +old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -name "*${SRCEXT}" -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages from the cleanup directory..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + ${SOURCE_CLEANUP_DRYRUN} || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + done +fi + +msg "Failed" +for _fail in ${failedpkgs[@]}; do + msg2 "$_fail" +done + + +script_unlock + diff --git a/db-functions b/db-functions index e2731cb..b900669 100644 --- a/db-functions +++ b/db-functions @@ -1,7 +1,5 @@ #!/bin/bash -. config - # Some PKGBUILDs need CARCH to be set CARCH="x86_64" @@ -18,7 +16,7 @@ restore_umask () { } # set up general environment -WORKDIR=$(mktemp -d ${TMPDIR}/$(basename $0).XXXXXXXXXX) +WORKDIR=$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX) LOCKS=() # check if messages are to be printed using color -- cgit v1.2.3-2-g168b From e2385920015311376c1631a42a1ca8f1c45dae4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 6 Mar 2011 18:08:25 -0600 Subject: * Fixed error that leaked non-free files to repo --- pato2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pato2.py b/pato2.py index 936e97a..336c6fb 100644 --- a/pato2.py +++ b/pato2.py @@ -56,7 +56,7 @@ def sync_all_repo(debug=verbose): cmd=generate_rsync_command(rsync_list_command) rsout=run_rsync(cmd) pkgs=pkginfo_from_rsync_output(rsout) - generate_exclude_list_from_blacklist(pkgs,listado(blacklist)) + generate_exclude_list_from_blacklist(pkgs,listado(blacklist),debug=False) cmd=generate_rsync_command(rsync_update_command,blacklist_file=blacklist) a=run_rsync(cmd) if debug: printf(a) -- cgit v1.2.3-2-g168b From 58a67fb771fbbf433fe3ab2dafb9194e6f635f13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 6 Mar 2011 18:09:52 -0600 Subject: * Added --delete to rsync-update-command until ftp-dir-cleanup can do delete --- config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.py b/config.py index 37bebc0..ada17f0 100644 --- a/config.py +++ b/config.py @@ -46,7 +46,7 @@ rsync_blacklist = docs + "/rsyncBlacklist" # Rsync commands rsync_list_command="rsync -a --no-motd --list-only " -rsync_update_command="rsync -av --delay-updates " +rsync_update_command="rsync -av --delete-after --delay-updates " # Classes and Exceptions class NonValidFile(ValueError): pass -- cgit v1.2.3-2-g168b From 125df5a815e0a327739e928cc765ffdcf4bd0aea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 6 Mar 2011 18:19:03 -0600 Subject: * Added function for cleanup dir --- pato2.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pato2.py b/pato2.py index 336c6fb..03afebd 100644 --- a/pato2.py +++ b/pato2.py @@ -129,6 +129,12 @@ def remove_from_blacklist(repo_,arch_,info_,blacklist_): a = commands.getoutput(com_) if verbose: printf(a) +def cleanup_nonfree_in_dir(directory,blacklisted_names): + pkgs=pkginfo_from_files_in_dir(directory) + for package in pkgs: + if package["name"] in blacklisted_names: + os.remove(package["location"]) + def link(repo_,arch_,file_): """ Makes a link in the repo for the package """ cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ -- cgit v1.2.3-2-g168b From 15ed2078e8743f87efcd63ac62cc0bfd5b39dc96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 6 Mar 2011 18:34:28 -0600 Subject: * Corrected import glob function --- filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filter.py b/filter.py index 2fe61f0..b6e8105 100644 --- a/filter.py +++ b/filter.py @@ -1,6 +1,6 @@ #! /usr/bin/python #-*- encoding: utf-8 -*- -import glob +from glob import glob from repm.config import * from repm.pato2 import * -- cgit v1.2.3-2-g168b From a41f5f44a8f812dfa2aacaf359bb51782d7a1633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 6 Mar 2011 19:02:14 -0600 Subject: * Fixed blacklist for rsync_command at sync_all_repo --- pato2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pato2.py b/pato2.py index 03afebd..18ace85 100644 --- a/pato2.py +++ b/pato2.py @@ -57,7 +57,7 @@ def sync_all_repo(debug=verbose): rsout=run_rsync(cmd) pkgs=pkginfo_from_rsync_output(rsout) generate_exclude_list_from_blacklist(pkgs,listado(blacklist),debug=False) - cmd=generate_rsync_command(rsync_update_command,blacklist_file=blacklist) + cmd=generate_rsync_command(rsync_update_command,blacklist_file=rsync_blacklist) a=run_rsync(cmd) if debug: printf(a) -- cgit v1.2.3-2-g168b From 0889827f252c9f6c21c1d4f2afc704b9b303e083 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Thu, 10 Mar 2011 13:59:35 -0600 Subject: * Fixed issue 71: pending list ends with newline --- pato2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pato2.py b/pato2.py index 18ace85..27a3962 100644 --- a/pato2.py +++ b/pato2.py @@ -111,7 +111,7 @@ def make_pending(repo_,arch_,info_): printf( pkg_ + " package has no %NAME% attibute " ) if verbose: printf( lista_ ) a=open( pending + "-" + repo_ + ".txt", "w" ).write( - "\n".join([name + ":" + license_ for (name,license_) in lista_]) ) + "\n".join([name + ":" + license_ for (name,license_) in lista_]) + "\n") def remove_from_blacklist(repo_,arch_,info_,blacklist_): """ Check the blacklist and remove packages on the db""" -- cgit v1.2.3-2-g168b From 47045b1934932b0695dd301a9c76b9dab1b03023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 21 Mar 2011 17:30:40 -0600 Subject: * Changed sync_all_repo to adress bug83 --- config.py | 3 ++- filter.py | 3 ++- pato2.py | 6 +++++- test/core.db.tar.gz | Bin 1637 -> 1345 bytes test/test_filter.py | 23 +++++++++++++++++++++++ 5 files changed, 32 insertions(+), 3 deletions(-) diff --git a/config.py b/config.py index ada17f0..c218ddd 100644 --- a/config.py +++ b/config.py @@ -46,7 +46,8 @@ rsync_blacklist = docs + "/rsyncBlacklist" # Rsync commands rsync_list_command="rsync -a --no-motd --list-only " -rsync_update_command="rsync -av --delete-after --delay-updates " +rsync_update_command="rsync -av --delay-updates --exclude='*.db.tar.{gz|xz}' " +rsync_post_command="rsync -av --delete " # Classes and Exceptions class NonValidFile(ValueError): pass diff --git a/filter.py b/filter.py index b6e8105..668822b 100644 --- a/filter.py +++ b/filter.py @@ -115,7 +115,8 @@ def pkginfo_from_files_in_dir(directory): package_list.append(pkginfo_from_filename(filename)) return tuple(package_list) - +def pkginfo_from_db(path_to_db): + """ """ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, exclude_file=rsync_blacklist, debug=verbose): diff --git a/pato2.py b/pato2.py index 27a3962..0d77d6b 100644 --- a/pato2.py +++ b/pato2.py @@ -59,7 +59,11 @@ def sync_all_repo(debug=verbose): generate_exclude_list_from_blacklist(pkgs,listado(blacklist),debug=False) cmd=generate_rsync_command(rsync_update_command,blacklist_file=rsync_blacklist) a=run_rsync(cmd) - if debug: printf(a) + cmd=generate_rsync_command(rsync_post_command,blacklist_file=rsync_blacklist) + b=run_rsync(cmd) + if debug: + printf(a) + printf(b) def get_from_desc(desc, var,db_tar_file=False): """ Get a var from desc file """ diff --git a/test/core.db.tar.gz b/test/core.db.tar.gz index a28ea64..5eb2081 100644 Binary files a/test/core.db.tar.gz and b/test/core.db.tar.gz differ diff --git a/test/test_filter.py b/test/test_filter.py index 5127b75..1906b87 100644 --- a/test/test_filter.py +++ b/test/test_filter.py @@ -163,7 +163,30 @@ class pkginfo_from_descKnownValues(unittest.TestCase): class pkginfo_from_db(unittest.TestCase): archdb = os.path.join("./workdir") + example_package_list=(Package(),Package(),Package()) + example_package_list[0].package_info={ "name" : "acl", + "version" : "2.2.49", + "release" : "2", + "arch" : "x86_64", + "license" : ("LGPL",), + "location": "acl-2.2.49-2-x86_64.pkg.tar.xz" + "depends" : ("attr>=2.4.41"),} + example_package_list[1].package_info={ "name" : "glibc", + "version" : "2.13", + "release" : "4", + "arch" : "x86_64", + "license" : ("GPL","LGPL"), + "location": "glibc-2.13-4-x86_64.pkg.tar.xz" + "depends" : ("linux-api-headers>=2.6.37","tzdata",),} + example_package_list[2].package_info={ "name" : "", + "version" : "2.2.26", + "release" : "1", + "arch" : "x86_64", + "license" : False, + "location": "" + "depends" : False,} if __name__ == "__main__": unittest.main() + -- cgit v1.2.3-2-g168b From fa11d216d567cec3c96c964829c51bd923b493c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 21 Mar 2011 17:49:24 -0600 Subject: * Don't update *.abs.tar.* --- config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index c218ddd..c643bb5 100644 --- a/config.py +++ b/config.py @@ -46,8 +46,8 @@ rsync_blacklist = docs + "/rsyncBlacklist" # Rsync commands rsync_list_command="rsync -a --no-motd --list-only " -rsync_update_command="rsync -av --delay-updates --exclude='*.db.tar.{gz|xz}' " -rsync_post_command="rsync -av --delete " +rsync_update_command="rsync -av --delay-updates --exclude=*.{abs|db}.tar.* " +rsync_post_command="rsync -av --delete --exclude=*.abs.tar.* " # Classes and Exceptions class NonValidFile(ValueError): pass -- cgit v1.2.3-2-g168b From 72d3e1102acf8e90a1646d5c9afa6fdd01b2aeca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 8 Apr 2011 11:40:32 -0500 Subject: Config ported to bash --- config.sh | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100755 config.sh diff --git a/config.sh b/config.sh new file mode 100755 index 0000000..aae3287 --- /dev/null +++ b/config.sh @@ -0,0 +1,51 @@ +#!/bin/sh +# -*- coding: utf-8 -*- + +# Mirror options +mirror="mirrors.eu.kernel.org" +mirrorpath="::mirrors/archlinux" + +# Directories and files + +## Optionals +paraboladir=~/parabolagnulinux.org +logtime=$(date -u +%Y%m%d-%H:%M) + +## Must be defined +logname=${paraboladir}/${logtime}-repo-maintainer.log +tempdir=~/tmp/ +docs_dir=${paraboladir}/docs +repodir=${paraboladir}/repo + +# Repos, arches, and dirs for repo +repolist="core:extra:community:testing:community-testing:multilib" +dir_list="pool" +arch_list="i686:x86_64" +other="any" + +# Output options +output="True" +debug="False" + +# Rsync commands +rsync_list_command="rsync -a --no-motd --list-only " +rsync_update_command="rsync -av --delay-updates --exclude=*.{abs|db}.tar.* " +rsync_post_command="rsync -av --delete --exclude=*.abs.tar.* " + + +function run_python_cmd { + env \ + mirror=${mirror} \ + mirrorpath=${mirrorpath} \ + logname=${logname} \ + tempdir=${tempdir} \ + docs_dir=${docs_dir} \ + repodir=${repodir} \ + repolist=${repolist} \ + dir_list=${dir_list} \ + arch_list=${arch_list} \ + other=${other} \ + output=${output} \ + debug=${debug} \ + $1 +} \ No newline at end of file -- cgit v1.2.3-2-g168b From 2b721124f5122faae5709e5e2dd8035f392c1361 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 10 Apr 2011 15:18:17 -0500 Subject: config.py import config.sh settings --- config.py | 72 ++++++++++++++++++++++++++++++--------------------------------- config.sh | 6 +++--- 2 files changed, 37 insertions(+), 41 deletions(-) diff --git a/config.py b/config.py index bd8ef1b..56a7c81 100644 --- a/config.py +++ b/config.py @@ -1,44 +1,36 @@ -#!/usr/bin/python +#!/usr/bin/pythonn # -*- coding: utf-8 -*- -from user import home -import commands +try: + from subprocess import check_output +except(ImportError): + from commands import getoutput as check_output +import os -time__ = commands.getoutput("date +%Y%m%d-%H:%M") +stringvars=("mirror", "mirrorpath", "logname", "tempdir", "docs_dir", + "repodir",) +listvars=("repo_list", "dir_list", "arch_list", "other",) +boolvars=("output", "debug",) -# Mirror Parameters -mirror = "mirrors.eu.kernel.org" -mirrorpath = "::mirrors/archlinux" +config=dict() -# Directories and files +def exit_if_none: + if os.environ.get(var) is None: + exit("%s is not defined" % var) -## Optionals -path = home + "/parabolagnulinux.org" -docs = path + "/docs" -logdir = path + "/log" - -## Must be defined -logname= logdir + "/" + time__ + "-repo-maintainer.log" -repodir= path + "/repo" -tmp = home + "/tmp" -archdb = tmp + "/db" - -free_path= path + "/free/" - -# Repo, arch, and other folders to use for repo -repo_list = ("core", "extra", "community", "testing", "community-testing", "multilib") -dir_list = ("pool","sources") -arch_list = ("i686", "x86_64") -other = ("any",) - -# Output -output = True -verbose = False - -# Files -blacklist = docs + "/blacklist.txt" -whitelist = docs + "/whitelist.txt" -pending = docs + "/pending" -rsync_blacklist = docs + "/rsyncBlacklist" +for var in stringvars: + exit_if_none(var) + config[var]=os.environ.get(var) +for var in listvars: + exit_if_none(var) + config[var]=tuple(os.environ.get(var).split(":")) +for var in boolvars: + exit_if_none(var) + if os.environ.get(var) == "True": + config[var]=True + elif os.environ.get(var) =="False": + config[var]=False + else: + print('%s is not True or False' % var) # Classes and Exceptions class NonValidFile(ValueError): pass @@ -55,7 +47,8 @@ class Package: "release" : False, "arch" : False, "license" : False, - "location": False} + "location": False, + "depends" : False,} def __setitem__(self, key, item): if key in self.package_info.keys(): @@ -76,8 +69,11 @@ class Package: if not isinstance(x, Package): return False for key in self.package_info.keys(): - if x[key] != self[key]: + if x[key] != self.package_info[key]: return False else: return True +if __name__=="__main__": + for key in config.keys(): + print("%s : %s" % (key,config[key])) diff --git a/config.sh b/config.sh index aae3287..66be477 100755 --- a/config.sh +++ b/config.sh @@ -1,6 +1,7 @@ #!/bin/sh # -*- coding: utf-8 -*- + # Mirror options mirror="mirrors.eu.kernel.org" mirrorpath="::mirrors/archlinux" @@ -18,7 +19,7 @@ docs_dir=${paraboladir}/docs repodir=${paraboladir}/repo # Repos, arches, and dirs for repo -repolist="core:extra:community:testing:community-testing:multilib" +repo_list="core:extra:community:testing:community-testing:multilib" dir_list="pool" arch_list="i686:x86_64" other="any" @@ -28,7 +29,6 @@ output="True" debug="False" # Rsync commands -rsync_list_command="rsync -a --no-motd --list-only " rsync_update_command="rsync -av --delay-updates --exclude=*.{abs|db}.tar.* " rsync_post_command="rsync -av --delete --exclude=*.abs.tar.* " @@ -41,7 +41,7 @@ function run_python_cmd { tempdir=${tempdir} \ docs_dir=${docs_dir} \ repodir=${repodir} \ - repolist=${repolist} \ + repo_list=${repo_list} \ dir_list=${dir_list} \ arch_list=${arch_list} \ other=${other} \ -- cgit v1.2.3-2-g168b From 69d84cc6edab249037522d00d3685939dc38495a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 10 Apr 2011 17:59:47 -0500 Subject: Firs main deliver --- config.py | 2 +- config.sh | 10 +++++--- libremessages | 77 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ main.sh | 21 ++++++++++++++++ 4 files changed, 106 insertions(+), 4 deletions(-) create mode 100755 libremessages create mode 100644 main.sh diff --git a/config.py b/config.py index 39f512a..8cf07a2 100644 --- a/config.py +++ b/config.py @@ -7,7 +7,7 @@ except(ImportError): import os stringvars=("mirror", "mirrorpath", "logname", "tempdir", "docs_dir", - "repodir",) + "repodir", "rsync_blacklist") listvars=("repo_list", "dir_list", "arch_list", "other",) boolvars=("output", "debug",) diff --git a/config.sh b/config.sh index 66be477..60bd4ea 100755 --- a/config.sh +++ b/config.sh @@ -17,6 +17,7 @@ logname=${paraboladir}/${logtime}-repo-maintainer.log tempdir=~/tmp/ docs_dir=${paraboladir}/docs repodir=${paraboladir}/repo +rsync_blacklist=${docs_dir}/rsyncBlacklist # Repos, arches, and dirs for repo repo_list="core:extra:community:testing:community-testing:multilib" @@ -29,8 +30,8 @@ output="True" debug="False" # Rsync commands -rsync_update_command="rsync -av --delay-updates --exclude=*.{abs|db}.tar.* " -rsync_post_command="rsync -av --delete --exclude=*.abs.tar.* " +rsync_update_command="rsync -av --delay-updates --exclude='*.{abs|db}.tar.*' " +rsync_post_command="rsync -av --delete --exclude='*.abs.tar.*' " function run_python_cmd { @@ -39,6 +40,7 @@ function run_python_cmd { mirrorpath=${mirrorpath} \ logname=${logname} \ tempdir=${tempdir} \ + rsync_blacklist=${rsync_blacklist} \ docs_dir=${docs_dir} \ repodir=${repodir} \ repo_list=${repo_list} \ @@ -48,4 +50,6 @@ function run_python_cmd { output=${output} \ debug=${debug} \ $1 -} \ No newline at end of file +} + +source libremessages \ No newline at end of file diff --git a/libremessages b/libremessages new file mode 100755 index 0000000..9fbbc2b --- /dev/null +++ b/libremessages @@ -0,0 +1,77 @@ +# Copyright (c) 2006-2010 Pacman Development Team +# Copyright (c) 2002-2006 by Judd Vinet +# Copyright (c) 2005 by Aurelien Foret +# Copyright (c) 2006 by Miklos Vajna +# Copyright (c) 2005 by Christian Hamar +# Copyright (c) 2006 by Alex Smith +# Copyright (c) 2006 by Andras Voroskoi +# Copyright (c) 2011 by Joshua Haase +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# gettext initialization +export TEXTDOMAIN='libretools' +export TEXTDOMAINDIR='/usr/share/locale' + +# check if messages are to be printed using color +unset ALL_OFF BOLD BLUE GREEN RED YELLOW + +if tput setaf 0 &>/dev/null; then + ALL_OFF="$(tput sgr0)" + BOLD="$(tput bold)" + BLUE="${BOLD}$(tput setaf 4)" + GREEN="${BOLD}$(tput setaf 2)" + RED="${BOLD}$(tput setaf 1)" + YELLOW="${BOLD}$(tput setaf 3)" + PURPLE="${ALL_OFF}$(tput setaf 5)" +else + ALL_OFF="\033[1;0m" + BOLD="\033[1;1m" + BLUE="${BOLD}\033[1;34m" + GREEN="${BOLD}\033[1;32m" + RED="${BOLD}\033[1;31m" + YELLOW="${BOLD}\033[1;33m" + PURPLE="${BOLD}\033[1;30;40m" +fi + +stdnull() { + local action=$1; + eval "${action} >/dev/null 2>&1" +} + +plain() { + local mesg=$1; shift + printf "${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +msg() { + local mesg=$1; shift + printf "${GREEN}==>${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +msg2() { + local mesg=$1; shift + printf "${BLUE} ->${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +warning() { + local mesg=$1; shift + printf "${YELLOW}==> $(gettext "WARNING:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +error() { + local mesg=$1; shift + printf "${RED}==> $(gettext "ERROR:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + diff --git a/main.sh b/main.sh new file mode 100644 index 0000000..ac504a4 --- /dev/null +++ b/main.sh @@ -0,0 +1,21 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +source config.sh + +function mkrsexclude { + error=1 + while ${error}; do + run_python_cmd "filter.py" + error=$? + done +} + +msg "Cleaning $tempdir" +stdnull "rm -r $tempdir/* " + +msg "Generating exclude list for rsync" +mkrsexclude + +msg "Syncing repo files without delete" +${rsync_update_command} --exclude-from=${rsync_blacklist} ${mirror}${mirropath}/ \ No newline at end of file -- cgit v1.2.3-2-g168b From 61ee5da8b31b44e80e008619a32ca886d8799646 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 10 Apr 2011 19:27:12 -0500 Subject: Replaced pato2.py functionality in main.sh --- main.sh | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/main.sh b/main.sh index ac504a4..2d59094 100644 --- a/main.sh +++ b/main.sh @@ -4,7 +4,7 @@ source config.sh function mkrsexclude { - error=1 + local error=1 while ${error}; do run_python_cmd "filter.py" error=$? @@ -17,5 +17,19 @@ stdnull "rm -r $tempdir/* " msg "Generating exclude list for rsync" mkrsexclude -msg "Syncing repo files without delete" -${rsync_update_command} --exclude-from=${rsync_blacklist} ${mirror}${mirropath}/ \ No newline at end of file +msg "Syncing repos without delete" +# rsync_update_command does not sync db or abs +${rsync_update_command} --exclude-from=${rsync_blacklist} \ + ${mirror}${mirropath}/{$(echo ${repo_list} | tr ':' ',')} ${repodir} + +msg "Syncing each repo and cleaning" +for repo in $(echo ${repo_list} | tr ':' ' '); do + msg2 "Syncing ${repo}" + ${rsync_post_command} --exclude-from=${rsync_blacklist} \ + ${mirror}${mirropath}/${repo} ${repodir}/${repo} + msg2 "Cleaning ${repo}" + clean-repo.py -d ${repodir}/${repo} \ + -b ${repodir}/${repo}/${repo}.db.tar.gz + msg2 "Making pending list for ${repo}" + run_python_cmd "mkpending.py -r ${repo} -d ${repodir}/${repo}" +done -- cgit v1.2.3-2-g168b From 3fb5f8efef231dd7784be880934cd106603ab6f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 11 Apr 2011 01:09:25 -0500 Subject: bash-port ready for testing. --- clean_repo.py | 45 +++++++++++ config.py | 10 ++- config.sh | 17 +++-- filter.py | 68 +++++++++++++---- get_license.sh | 10 +-- main.sh | 20 +++-- mkpending.py | 45 +++++++++++ pato2.py | 209 ++++------------------------------------------------ test/test_filter.py | 6 +- 9 files changed, 198 insertions(+), 232 deletions(-) create mode 100644 clean_repo.py create mode 100644 mkpending.py diff --git a/clean_repo.py b/clean_repo.py new file mode 100644 index 0000000..29d446d --- /dev/null +++ b/clean_repo.py @@ -0,0 +1,45 @@ +#! /usr/bin/python +#-*- encoding: utf-8 -*- +from repm.filter import * +import argparse + +def remove_from_blacklist(path_to_db, blacklisted_names, + debug=config["debug"]): + """ Check the blacklist and remove packages on the db""" + + pkgs=[pkg for pkg in pkginfo_from_db(path_to_db) if + pkg["name"] in blacklisted_names] + if pkgs: + lista=" ".join(pkgs) + cmd = "repo-remove " + path_to_db + " " + lista + printf(cmd) + a = check_output(cmd) + if debug: + printf(a) + return pkgs, cmd + +def cleanup_nonfree_in_dir(directory, blacklisted_names): + pkgs=pkginfo_from_files_in_dir(directory) + for package in pkgs: + if package["name"] in blacklisted_names: + os.remove(package["location"]) + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Clean a repo db and packages") + parser.add_argument("-b", "--database", type=str, + help="dabatase to clean") + parser.add_argument("-d", "--directory", type=str, + help="directory to clean") + args=parser.parse_args() + + if args.directory: + cleanup_nonfree_in_dir(args.database, listado(config["blacklist"])) + + if args.database: + pkgs=pkginfo_from_db(args.database) + remove_from_blacklist(args.database, pkgs, + tuple(listado(config["blacklist"]) + + listado(config["pending"]))) + if not args.directory and not args.database: + parser.print_help() diff --git a/config.py b/config.py index 8cf07a2..24ecfaf 100644 --- a/config.py +++ b/config.py @@ -1,4 +1,4 @@ -#!/usr/bin/pythonn +#!/usr/bin/python # -*- coding: utf-8 -*- try: from subprocess import check_output @@ -7,22 +7,24 @@ except(ImportError): import os stringvars=("mirror", "mirrorpath", "logname", "tempdir", "docs_dir", - "repodir", "rsync_blacklist") + "repodir", "rsync_blacklist") listvars=("repo_list", "dir_list", "arch_list", "other",) boolvars=("output", "debug",) config=dict() -def exit_if_none: +def exit_if_none(var): if os.environ.get(var) is None: exit("%s is not defined" % var) for var in stringvars: exit_if_none(var) - config[var]=os.environ.get(var) + config[var]=os.environ.get(var) + for var in listvars: exit_if_none(var) config[var]=tuple(os.environ.get(var).split(":")) + for var in boolvars: exit_if_none(var) if os.environ.get(var) == "True": diff --git a/config.sh b/config.sh index 60bd4ea..741dee4 100755 --- a/config.sh +++ b/config.sh @@ -1,22 +1,25 @@ #!/bin/sh # -*- coding: utf-8 -*- - # Mirror options mirror="mirrors.eu.kernel.org" mirrorpath="::mirrors/archlinux" -# Directories and files - +# Directories ## Optionals paraboladir=~/parabolagnulinux.org logtime=$(date -u +%Y%m%d-%H:%M) - ## Must be defined logname=${paraboladir}/${logtime}-repo-maintainer.log tempdir=~/tmp/ docs_dir=${paraboladir}/docs repodir=${paraboladir}/repo +# End Directories + +# Files +blacklist=${docs_dir}/blacklist.txt +whitelist=${docs_dir}/whitelist.txt +pending=${docs_dir}/pending rsync_blacklist=${docs_dir}/rsyncBlacklist # Repos, arches, and dirs for repo @@ -33,16 +36,18 @@ debug="False" rsync_update_command="rsync -av --delay-updates --exclude='*.{abs|db}.tar.*' " rsync_post_command="rsync -av --delete --exclude='*.abs.tar.*' " - function run_python_cmd { env \ mirror=${mirror} \ mirrorpath=${mirrorpath} \ logname=${logname} \ tempdir=${tempdir} \ - rsync_blacklist=${rsync_blacklist} \ docs_dir=${docs_dir} \ repodir=${repodir} \ + blacklist=${blacklist} \ + whitelist=${whitelist} \ + pending=${pending} \ + rsync_blacklist=${rsync_blacklist} \ repo_list=${repo_list} \ dir_list=${dir_list} \ arch_list=${arch_list} \ diff --git a/filter.py b/filter.py index 668822b..1a0fa6f 100644 --- a/filter.py +++ b/filter.py @@ -4,6 +4,13 @@ from glob import glob from repm.config import * from repm.pato2 import * +def listado(filename): + """Obtiene una lista de paquetes de un archivo.""" + archivo = open(filename,"r") + lista = archivo.read().split("\n") + archivo.close() + return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] + def pkginfo_from_filename(filename): """ Generates a Package object with info from a filename, filename can be relative or absolute @@ -116,10 +123,48 @@ def pkginfo_from_files_in_dir(directory): return tuple(package_list) def pkginfo_from_db(path_to_db): - """ """ + """ Get PKGINFO from db. + + Parameters: + ---------- + path_to_db -> str Path to file -def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, - exclude_file=rsync_blacklist, debug=verbose): + Output: + ---------- + None """ + package_list=list() + + if not os.path.isfile(path_to_db): + raise NonValidFile(path_to_db + "is not a file") + + check_output("mkdir -p " + archdb) + + try: + db_open_tar = tarfile.open(db_tar_file, 'r:gz') + except tarfile.ReadError: + printf("No valid db_file %s or not readable" % db_tar_file) + return(tuple()) + else: + printf("No db_file %s" % db_tar_file) + return(tuple()) + + for file in db_open_tar.getmembers(): + db_open_tar.extract(file, archdb) + db_open_tar.close() + # Get info from file + for dir_ in glob(archdb + "/*"): + if isdir(dir_) and isfile(dir_ + "/desc"): + package_list.append(pkginfo_from_desc( + os.path.join(dir_,"desc"))) + check_output("rm -r %s/*" % archdb) + if verbose_: + printf(package_list) + return package_list + +def generate_exclude_list_from_blacklist(packages_iterable, + blacklisted_names, + exclude_file=config["rsync_blacklist"], + debug=config["debug"]): """ Generate an exclude list for rsync Parameters: @@ -132,16 +177,12 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, Output: ---------- None """ - a=list() - - for package in packages_iterable: - if not isinstance(package, Package): - raise ValueError(" %s is not a Package object " % package) - if package["name"] in blacklisted_names: - a.append(package["location"]) + pkgs=[pkg["location"] for pkg in packages_iterable + if isinstance(pkg, Package) + and pkg["name"] in blacklisted_names] if debug: - return a + return pkgs try: fsock = open(exclude_file,"w") try: @@ -149,9 +190,10 @@ def generate_exclude_list_from_blacklist(packages_iterable, blacklisted_names, finally: fsock.close() except IOError: - printf("%s wasnt written" % blacklist_file) + printf("%s wasnt written" % exclude_file) if __name__ == "__main__": - a=run_rsync(rsync_list_command) + cmd=generate_rsync_command(rsync_list_command) + a=run_rsync(cmd) packages=pkginfo_from_rsync_output(a) generate_exclude_list_from_blacklist(packages,listado(blacklist)) diff --git a/get_license.sh b/get_license.sh index a7241a1..0da58cb 100755 --- a/get_license.sh +++ b/get_license.sh @@ -31,12 +31,12 @@ rm -rf $dir/* tempdir=$(mktemp -d) cd $tempdir -a=($(cut -d: -f1 $docs/pending*.txt)) -echo ${a[@]} +pending=($(cut -d: -f1 $docs/pending*.txt)) +echo ${pending[@]} -for x in ${a[@]}; do - b=( $(ls $repo/*/os/*/$x*) ) - for y in ${b[@]}; do +for pkg in ${pending[@]}; do + pkg_in_repo=( $(ls ${repo}/*/os/*/${pkg}*) ) + for y in ${pkg_in_repo[@]}; do echo "chmod +r $y" chmod +r $y echo "tar -xf $y usr/share/licenses" diff --git a/main.sh b/main.sh index 2d59094..1a2c6c4 100644 --- a/main.sh +++ b/main.sh @@ -23,13 +23,17 @@ ${rsync_update_command} --exclude-from=${rsync_blacklist} \ ${mirror}${mirropath}/{$(echo ${repo_list} | tr ':' ',')} ${repodir} msg "Syncing each repo and cleaning" +msg2 "Remove pending files" +stdnull "rm -rf ${pending}*" for repo in $(echo ${repo_list} | tr ':' ' '); do - msg2 "Syncing ${repo}" - ${rsync_post_command} --exclude-from=${rsync_blacklist} \ - ${mirror}${mirropath}/${repo} ${repodir}/${repo} - msg2 "Cleaning ${repo}" - clean-repo.py -d ${repodir}/${repo} \ - -b ${repodir}/${repo}/${repo}.db.tar.gz - msg2 "Making pending list for ${repo}" - run_python_cmd "mkpending.py -r ${repo} -d ${repodir}/${repo}" + for arch in $(echo ${arch_list} | tr ':' ' '); do + msg2 "Syncing ${repo} ${arch}" + ${rsync_post_command} --exclude-from=${rsync_blacklist} \ + ${mirror}${mirropath}/${repo} ${repodir}/${repo} + msg2 "Making pending list for ${repo} ${arch}" + run_python_cmd "mkpending.py -r ${repo} -b ${repodir}/${repo}/os/${arch}" + msg2 "Cleaning ${repo} ${arch}" + run_python_cmd "clean-repo.py -b ${repodir}/${repo}/os/${arch}/${repo}.db.tar.gz -d ${repodir}/${repo}/os/${arch}/" + get_license.sh + done done diff --git a/mkpending.py b/mkpending.py new file mode 100644 index 0000000..43a5fb2 --- /dev/null +++ b/mkpending.py @@ -0,0 +1,45 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +from repm.filter import * + +def make_pending(path_to_db): + """ Determine wich packages are pending for license auditing.""" + packages_iterable=pkginfo_from_db(path_to_db) + search = tuple(listado(config["blacklist"]) + + listado(config["whitelist"])) + + pkgs=[pkg for pkg in packages_iterable + if "custom" in pkg["license"] + and pkg["name"] not in search] + return pkgs + +def write_pending(packages_iterable, repo, prefix=config["pending"]): + """ Write a pending file with the info of the packages """ + filename=prefix + "-" + repo + ".txt" + try: + fsock=open(filename, "a") + except(IOError): + print("Can't read %s" % filename) + finally: + fsock.close() + if os.path.isfile(filename): + pkgs=[pkg for pkg in packages_iterable if pkg["name"] not in + listado(filename)] + fsock.write("\n".join([pkg["name"] + ":" + pkg["license"] + for pkg in pkgs]) + "\n") + fsock.close() + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Clean a repo db and packages") + parser.add_argument("-b", "--dababase", type=str, required=True + help="database to check") + parser.add_argument("-r", "--repo", type=str, required=True + help="repo of database") + args=parser.parse_args() + + if args.database and args.repo: + pkgs=make_pending(args.database) + write_pending(pkgs, args.repo) + else: + parser.print_help() diff --git a/pato2.py b/pato2.py index 0d77d6b..4cdb536 100644 --- a/pato2.py +++ b/pato2.py @@ -27,159 +27,23 @@ from repm.config import * from repm.filter import * import tarfile -from glob import glob from os.path import isdir, isfile -def printf(text,output_=output): +def printf(text,output=config["output"]): """Guarda el texto en la variable log y puede imprimir en pantalla.""" - log_file = open(logname, 'a') + log_file = open(config["logname"], 'a') log_file.write("\n" + str(text) + "\n") log_file.close() - if output_: print (str(text) + "\n") - -def listado(filename_): - """Obtiene una lista de paquetes de un archivo.""" - archivo = open(filename_,"r") - lista = archivo.read().split("\n") - archivo.close() - return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] - -def db(repo_,arch_): - """Construye un nombre para sincronizar una base de datos.""" - return "/%s/os/%s/%s.db.tar.gz" % (repo_, arch_, repo_) - -def packages(repo_, arch_, expr="*"): - """ Get packages on a repo, arch folder """ - return tuple( glob( repodir + "/" + repo_ + "/os/" + arch_ + "/" + expr ) ) - -def sync_all_repo(debug=verbose): - cmd=generate_rsync_command(rsync_list_command) - rsout=run_rsync(cmd) - pkgs=pkginfo_from_rsync_output(rsout) - generate_exclude_list_from_blacklist(pkgs,listado(blacklist),debug=False) - cmd=generate_rsync_command(rsync_update_command,blacklist_file=rsync_blacklist) - a=run_rsync(cmd) - cmd=generate_rsync_command(rsync_post_command,blacklist_file=rsync_blacklist) - b=run_rsync(cmd) - if debug: - printf(a) - printf(b) - -def get_from_desc(desc, var,db_tar_file=False): - """ Get a var from desc file """ - desc = desc.split("\n") - return desc[desc.index(var)+1] - -def get_info(repo_,arch_,db_tar_file=False,verbose_=verbose): - """ Makes a list of package name, file and license """ - info=list() - # Extract DB tar.gz - commands.getoutput("mkdir -p " + archdb) - if not db_tar_file: - db_tar_file = repodir + db(repo_,arch_) - if isfile(db_tar_file): - try: - db_open_tar = tarfile.open(db_tar_file, 'r:gz') - except tarfile.ReadError: - printf("No valid db_file %s" % db_tar_file) - return(tuple()) - else: - printf("No db_file %s" % db_tar_file) - return(tuple()) - for file in db_open_tar.getmembers(): - db_open_tar.extract(file, archdb) - db_open_tar.close() - # Get info from file - for dir_ in glob(archdb + "/*"): - if isdir(dir_) and isfile(dir_ + "/desc"): - pkg_desc_file = open(dir_ + "/desc", "r") - desc = pkg_desc_file.read() - pkg_desc_file.close() - info.append(( get_from_desc(desc,"%NAME%"), - dir_.split("/")[-1], - get_from_desc(desc,"%LICENSE%") )) - if verbose_: printf(info) - commands.getoutput("rm -r %s/*" % archdb) - return tuple(info) - -def make_pending(repo_,arch_,info_): - """ Si los paquetes no están en blacklist ni whitelist y la licencia contiene "custom" los agrega a pending""" - search = tuple( listado(blacklist) + listado (whitelist) ) - if verbose: printf("blaclist + whitelist= " + str(search) ) - lista_=list() - for (name,pkg_,license_) in info_: - if "custom" in license_: - if name not in search: - lista_.append( (name, license_ ) ) - elif not name: - printf( pkg_ + " package has no %NAME% attibute " ) - if verbose: printf( lista_ ) - a=open( pending + "-" + repo_ + ".txt", "w" ).write( - "\n".join([name + ":" + license_ for (name,license_) in lista_]) + "\n") - -def remove_from_blacklist(repo_,arch_,info_,blacklist_): - """ Check the blacklist and remove packages on the db""" - lista_=list() - pack_=list() - for (name_, pkg_, license_) in info_: - if name_ in blacklist_: - lista_.append(name_) - for p in packages(repo_,arch_,pkg_ + "*"): - pack_.append(p) - if lista_: - lista_=" ".join(lista_) - com_ = "repo-remove " + repodir + db(repo_,arch_) + " " + lista_ - printf(com_) - a = commands.getoutput(com_) - if verbose: printf(a) - -def cleanup_nonfree_in_dir(directory,blacklisted_names): - pkgs=pkginfo_from_files_in_dir(directory) - for package in pkgs: - if package["name"] in blacklisted_names: - os.remove(package["location"]) - -def link(repo_,arch_,file_): - """ Makes a link in the repo for the package """ - cmd_="ln -f " + file_ + " " + repodir + "/" + repo_ + "/os/" + arch_ - a=commands.getoutput(cmd_) - if verbose: - printf(cmd_ + a) - -def add_free_repo(verbose_=verbose): - cmd_=os.path.join(home,"/usr/bin/sync-free") - printf(cmd_) - a=commands.getoutput(cmd_) - if verbose_: printf(a) - for repo_ in repo_list: - for arch_ in arch_list: - lista_=list() - for file_ in glob(freedir + repo_ + "/os/" + arch_ + "/*.pkg.tar.*"): - lista_.append(file_) - for dir_ in other: - for file_ in glob(freedir + repo_ + "/os/" + dir_ + "/*.pkg.tar.*"): - lista_.append(file_) - - printf(lista_) - - if lista_: - lista_=" ".join(lista_) - if verbose: printf(lista_) - cmd_="repo-add " + repodir + db(repo_,arch_) + " " + lista_ - printf(cmd_) - a=commands.getoutput(cmd_) - if verbose: printf(a) - -def get_licenses(verbose_=verbose): - """ Extract the license from packages in repo_,arch_ and in pending_ file""" - cmd_=home + "/usr/bin/get_license.sh" - printf(cmd_) - a=commands.getoutput(cmd_) - if verbose_: printf(a) - -def generate_rsync_command(base_command, dir_list=(repo_list + dir_list), destdir=repodir, - source=mirror+mirrorpath, blacklist_file=False): - """ Generates an rsync command for executing it by combining all parameters. + if output_: + print (str(text) + "\n") + +def generate_rsync_command(base_command, + dir_list=(config["repo_list"] + + config["dir_list"]), + destdir=config["repodir"], + source=config["mirror"] +config["mirrorpath"]): + """ Generates an rsync command for executing + it by combining all parameters. Parameters: ---------- @@ -192,57 +56,16 @@ def generate_rsync_command(base_command, dir_list=(repo_list + dir_list), destdi Return: ---------- rsync_command -> str """ - from os.path import isfile, isdir - - if blacklist_file and not isfile(blacklist_file): - print(blacklist_file + " is not a file") - raise NonValidFile - if not os.path.isdir(destdir): print(destdir + " is not a directory") raise NonValidDir dir_list="{" + ",".join(dir_list) + "}" + return " ".join((base_command, os.path.join(source, dir_list), + destdir)) - if blacklist_file: - return " ".join((base_command, "--exclude-from="+blacklist_file, - os.path.join(source, dir_list), destdir)) - return " ".join((base_command, os.path.join(source, dir_list), destdir)) - -def run_rsync(command,debug=verbose): +def run_rsync(command,debug=config["debug"]): """ Runs rsync and gets returns it's output """ if debug: printf("rsync_command: " + command) - return commands.getoutput(command) - -if __name__ == "__main__": - from time import time - start_time = time() - def minute(): - return str(round((time() - start_time)/60, 1)) - - printf(" Cleaning %s folder " % (tmp) ) - commands.getoutput("rm -r %s/*" % tmp) - printf(" Syncing repo") - sync_all_repo(True) - - printf(" Updating databases and pending files lists: minute %s \n" % minute() ) - for repo in repo_list: - for arch in arch_list: - printf( "\n" + repo + "-" + arch + "\n" ) - printf( "Get info: minute %s " % minute() ) - info=get_info(repo,arch) - printf( "Make pending: minute %s" % minute() ) - make_pending(repo,arch,info) - printf( "Update DB: minute %s" % minute() ) - remove_from_blacklist( - repo, arch, info, tuple( listado(blacklist) + listado(pending + "-" + repo + ".txt") ) ) - - printf("Adding Parabola Packages: minute %s\n" % minute() ) - add_free_repo(True) - - printf("Extracting licenses in pending: minute %s" % minute() ) - get_licenses() - - printf("\n\nDelay: %s minutes \n" % minute()) - + return check_output(command) diff --git a/test/test_filter.py b/test/test_filter.py index 1906b87..5601d57 100644 --- a/test/test_filter.py +++ b/test/test_filter.py @@ -169,21 +169,21 @@ class pkginfo_from_db(unittest.TestCase): "release" : "2", "arch" : "x86_64", "license" : ("LGPL",), - "location": "acl-2.2.49-2-x86_64.pkg.tar.xz" + "location": "acl-2.2.49-2-x86_64.pkg.tar.xz", "depends" : ("attr>=2.4.41"),} example_package_list[1].package_info={ "name" : "glibc", "version" : "2.13", "release" : "4", "arch" : "x86_64", "license" : ("GPL","LGPL"), - "location": "glibc-2.13-4-x86_64.pkg.tar.xz" + "location": "glibc-2.13-4-x86_64.pkg.tar.xz", "depends" : ("linux-api-headers>=2.6.37","tzdata",),} example_package_list[2].package_info={ "name" : "", "version" : "2.2.26", "release" : "1", "arch" : "x86_64", "license" : False, - "location": "" + "location": "", "depends" : False,} -- cgit v1.2.3-2-g168b From e0e4837f17f22b7bfafafc8d0e6f7a351249417e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 11 Apr 2011 01:19:32 -0500 Subject: FixedFixed a little main.sh --- main.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/main.sh b/main.sh index 1a2c6c4..cf7bcd8 100644 --- a/main.sh +++ b/main.sh @@ -33,7 +33,9 @@ for repo in $(echo ${repo_list} | tr ':' ' '); do msg2 "Making pending list for ${repo} ${arch}" run_python_cmd "mkpending.py -r ${repo} -b ${repodir}/${repo}/os/${arch}" msg2 "Cleaning ${repo} ${arch}" - run_python_cmd "clean-repo.py -b ${repodir}/${repo}/os/${arch}/${repo}.db.tar.gz -d ${repodir}/${repo}/os/${arch}/" - get_license.sh + run_python_cmd "clean_repo.py -b ${repodir}/${repo}/os/${arch}/${repo}.db.tar.gz -d ${repodir}/${repo}/os/${arch}/" done done + +msg "Checking licenses" +get_license.sh -- cgit v1.2.3-2-g168b From f6184ee3c9422c40effb0265527ee5a31b027a1a Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernandez Date: Sun, 10 Apr 2011 23:44:01 -0700 Subject: chmod +x --- .gitignore | 3 ++- clean_repo.py | 0 config.py | 0 filter.py | 0 main.sh | 2 +- mkpending.py | 0 6 files changed, 3 insertions(+), 2 deletions(-) mode change 100644 => 100755 clean_repo.py mode change 100644 => 100755 config.py mode change 100644 => 100755 filter.py mode change 100644 => 100755 main.sh mode change 100644 => 100755 mkpending.py diff --git a/.gitignore b/.gitignore index e645833..cfefcc3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *~ -*.pyc \ No newline at end of file +*.pyc +cptobin.sh diff --git a/clean_repo.py b/clean_repo.py old mode 100644 new mode 100755 diff --git a/config.py b/config.py old mode 100644 new mode 100755 diff --git a/filter.py b/filter.py old mode 100644 new mode 100755 diff --git a/main.sh b/main.sh old mode 100644 new mode 100755 index cf7bcd8..886ea38 --- a/main.sh +++ b/main.sh @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/bin/bash # -*- coding: utf-8 -*- source config.sh diff --git a/mkpending.py b/mkpending.py old mode 100644 new mode 100755 -- cgit v1.2.3-2-g168b From a5f24954011da556977859d8a306ddb39c1ec2e4 Mon Sep 17 00:00:00 2001 From: Joshua Haase Date: Sun, 10 Apr 2011 23:49:50 -0700 Subject: Added cptobin --- .gitignore | 2 +- cptobin.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100755 cptobin.sh diff --git a/.gitignore b/.gitignore index cfefcc3..eef29c1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ *~ *.pyc -cptobin.sh + diff --git a/cptobin.sh b/cptobin.sh new file mode 100755 index 0000000..25a22d4 --- /dev/null +++ b/cptobin.sh @@ -0,0 +1,2 @@ +#!/bin/bash +cp -f clean_repo.py config.py config.sh main.sh mkpending.py filter.py ~/usr/bin/ \ No newline at end of file -- cgit v1.2.3-2-g168b From 67038ba1840d0f57b0ce49fdabd3dfa8057e2451 Mon Sep 17 00:00:00 2001 From: Joshua Haase Date: Mon, 11 Apr 2011 00:22:06 -0700 Subject: fixed some errors --- clean_repo.py | 2 +- config.py | 4 ++-- config.sh | 2 ++ cptobin.sh | 2 +- filter.py | 8 ++++---- mkpending.py | 4 ++-- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/clean_repo.py b/clean_repo.py index 29d446d..b0d306f 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -34,7 +34,7 @@ if __name__ == "__main__": args=parser.parse_args() if args.directory: - cleanup_nonfree_in_dir(args.database, listado(config["blacklist"])) + cleanup_nonfree_in_dir(args.directory, listado(config["blacklist"])) if args.database: pkgs=pkginfo_from_db(args.database) diff --git a/config.py b/config.py index 24ecfaf..f4089f9 100755 --- a/config.py +++ b/config.py @@ -6,8 +6,8 @@ except(ImportError): from commands import getoutput as check_output import os -stringvars=("mirror", "mirrorpath", "logname", "tempdir", "docs_dir", - "repodir", "rsync_blacklist") +stringvars=("mirror", "mirrorpath", "logname", "tempdir", "archdb", "docs_dir", + "repodir", "blacklist", "whitelist", "pending", "rsync_blacklist") listvars=("repo_list", "dir_list", "arch_list", "other",) boolvars=("output", "debug",) diff --git a/config.sh b/config.sh index 741dee4..4f42e0b 100755 --- a/config.sh +++ b/config.sh @@ -12,6 +12,7 @@ logtime=$(date -u +%Y%m%d-%H:%M) ## Must be defined logname=${paraboladir}/${logtime}-repo-maintainer.log tempdir=~/tmp/ +archdb=${tempdir}/db docs_dir=${paraboladir}/docs repodir=${paraboladir}/repo # End Directories @@ -42,6 +43,7 @@ function run_python_cmd { mirrorpath=${mirrorpath} \ logname=${logname} \ tempdir=${tempdir} \ + archdb=${archdb} \ docs_dir=${docs_dir} \ repodir=${repodir} \ blacklist=${blacklist} \ diff --git a/cptobin.sh b/cptobin.sh index 25a22d4..e803132 100755 --- a/cptobin.sh +++ b/cptobin.sh @@ -1,2 +1,2 @@ #!/bin/bash -cp -f clean_repo.py config.py config.sh main.sh mkpending.py filter.py ~/usr/bin/ \ No newline at end of file +cp -f clean_repo.py config.py config.sh get_license.sh main.sh mkpending.py filter.py ~/usr/bin/ \ No newline at end of file diff --git a/filter.py b/filter.py index 1a0fa6f..78ad410 100755 --- a/filter.py +++ b/filter.py @@ -137,15 +137,15 @@ def pkginfo_from_db(path_to_db): if not os.path.isfile(path_to_db): raise NonValidFile(path_to_db + "is not a file") - check_output("mkdir -p " + archdb) + check_output("mkdir -p " + config["archdb"]) try: - db_open_tar = tarfile.open(db_tar_file, 'r:gz') + db_open_tar = tarfile.open(path_to_db, 'r:gz') except tarfile.ReadError: - printf("No valid db_file %s or not readable" % db_tar_file) + printf("No valid db_file %s or not readable" % path_to_db) return(tuple()) else: - printf("No db_file %s" % db_tar_file) + printf("No db_file %s" % path_to_db) return(tuple()) for file in db_open_tar.getmembers(): diff --git a/mkpending.py b/mkpending.py index 43a5fb2..2b255a8 100755 --- a/mkpending.py +++ b/mkpending.py @@ -32,9 +32,9 @@ def write_pending(packages_iterable, repo, prefix=config["pending"]): if __name__ == "__main__": parser = argparse.ArgumentParser( description="Clean a repo db and packages") - parser.add_argument("-b", "--dababase", type=str, required=True + parser.add_argument("-b", "--dababase", type=str, required=True, help="database to check") - parser.add_argument("-r", "--repo", type=str, required=True + parser.add_argument("-r", "--repo", type=str, required=True, help="repo of database") args=parser.parse_args() -- cgit v1.2.3-2-g168b From 748600bf8dfba34b336ad642a6b26dae674db85f Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 11 Apr 2011 00:25:53 -0700 Subject: Will rsync be better now? --- config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index c643bb5..3e00eb0 100644 --- a/config.py +++ b/config.py @@ -46,8 +46,8 @@ rsync_blacklist = docs + "/rsyncBlacklist" # Rsync commands rsync_list_command="rsync -a --no-motd --list-only " -rsync_update_command="rsync -av --delay-updates --exclude=*.{abs|db}.tar.* " -rsync_post_command="rsync -av --delete --exclude=*.abs.tar.* " +rsync_update_command="rsync -av --delay-updates --exclude='*.{abs|db}.tar.*' " +rsync_post_command="rsync -av --delete --exclude='*.abs.tar.*' " # Classes and Exceptions class NonValidFile(ValueError): pass -- cgit v1.2.3-2-g168b From 857ecbe794c714919612b533303e7a9ef781c75b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 11 Apr 2011 10:26:40 -0500 Subject: Use non tracked file local_config for config. --- .gitignore | 2 +- config.sh | 33 +-------------------------------- local_config.example | 31 +++++++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 33 deletions(-) create mode 100644 local_config.example diff --git a/.gitignore b/.gitignore index eef29c1..69dec40 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ *~ *.pyc - +local_config diff --git a/config.sh b/config.sh index 4f42e0b..75d4b73 100755 --- a/config.sh +++ b/config.sh @@ -1,37 +1,6 @@ #!/bin/sh # -*- coding: utf-8 -*- - -# Mirror options -mirror="mirrors.eu.kernel.org" -mirrorpath="::mirrors/archlinux" - -# Directories -## Optionals -paraboladir=~/parabolagnulinux.org -logtime=$(date -u +%Y%m%d-%H:%M) -## Must be defined -logname=${paraboladir}/${logtime}-repo-maintainer.log -tempdir=~/tmp/ -archdb=${tempdir}/db -docs_dir=${paraboladir}/docs -repodir=${paraboladir}/repo -# End Directories - -# Files -blacklist=${docs_dir}/blacklist.txt -whitelist=${docs_dir}/whitelist.txt -pending=${docs_dir}/pending -rsync_blacklist=${docs_dir}/rsyncBlacklist - -# Repos, arches, and dirs for repo -repo_list="core:extra:community:testing:community-testing:multilib" -dir_list="pool" -arch_list="i686:x86_64" -other="any" - -# Output options -output="True" -debug="False" +source local_config # Rsync commands rsync_update_command="rsync -av --delay-updates --exclude='*.{abs|db}.tar.*' " diff --git a/local_config.example b/local_config.example new file mode 100644 index 0000000..7edfde7 --- /dev/null +++ b/local_config.example @@ -0,0 +1,31 @@ +# Mirror options +mirror="mirrors.eu.kernel.org" +mirrorpath="::mirrors/archlinux" + +# Directories +## Optionals +paraboladir=~/parabolagnulinux.org +logtime=$(date -u +%Y%m%d-%H:%M) +## Must be defined +logname=${paraboladir}/${logtime}-repo-maintainer.log +tempdir=~/tmp/ +archdb=${tempdir}/db +docs_dir=${paraboladir}/docs +repodir=${paraboladir}/repo +# End Directories + +# Files +blacklist=${docs_dir}/blacklist.txt +whitelist=${docs_dir}/whitelist.txt +pending=${docs_dir}/pending +rsync_blacklist=${docs_dir}/rsyncBlacklist + +# Repos, arches, and dirs for repo +repo_list="core:extra:community:testing:community-testing:multilib" +dir_list="pool" +arch_list="i686:x86_64" +other="any" + +# Output options +output="True" +debug="False" -- cgit v1.2.3-2-g168b From 3e27d11f68571bce92138f6cbfcaecac75fa1644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 11 Apr 2011 12:39:40 -0500 Subject: Fixed some errors --- clean_repo.py | 6 +++++- config.py | 35 +++++++++++++++++++++++++++++------ config.sh | 5 ----- cptobin.sh | 2 +- filter.py | 32 +++++++++++++++++--------------- local_config.example | 10 +++++++--- main.sh | 3 ++- mkpending.py | 35 ++++++++++++++++++++--------------- pato2.py | 10 +--------- 9 files changed, 82 insertions(+), 56 deletions(-) diff --git a/clean_repo.py b/clean_repo.py index b0d306f..d4e06fc 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -6,7 +6,9 @@ import argparse def remove_from_blacklist(path_to_db, blacklisted_names, debug=config["debug"]): """ Check the blacklist and remove packages on the db""" - + if "~" in path_to_db: + path_to_db=(os.path.expanduser(path_to_db)) + pkgs=[pkg for pkg in pkginfo_from_db(path_to_db) if pkg["name"] in blacklisted_names] if pkgs: @@ -19,6 +21,8 @@ def remove_from_blacklist(path_to_db, blacklisted_names, return pkgs, cmd def cleanup_nonfree_in_dir(directory, blacklisted_names): + if "~" in directory: + directory=(os.path.expanduser(directory)) pkgs=pkginfo_from_files_in_dir(directory) for package in pkgs: if package["name"] in blacklisted_names: diff --git a/config.py b/config.py index f4089f9..8b48c66 100755 --- a/config.py +++ b/config.py @@ -3,11 +3,15 @@ try: from subprocess import check_output except(ImportError): - from commands import getoutput as check_output + from commands import getoutput + def check_output(*popenargs,**kwargs): + cmd=" ".join(*popenargs) + return getoutput(cmd) import os -stringvars=("mirror", "mirrorpath", "logname", "tempdir", "archdb", "docs_dir", - "repodir", "blacklist", "whitelist", "pending", "rsync_blacklist") +stringvars=("mirror", "mirrorpath", "logname", "tempdir", "archdb", + "repodir", "blacklist", "whitelist", "pending", + "rsync_blacklist",) listvars=("repo_list", "dir_list", "arch_list", "other",) boolvars=("output", "debug",) @@ -37,10 +41,29 @@ for var in boolvars: # Rsync commands rsync_list_command="rsync -a --no-motd --list-only " +def printf(text,output=config["output"]): + """Guarda el texto en la variable log y puede imprimir en pantalla.""" + log_file = open(config["logname"], 'a') + log_file.write("\n" + str(text) + "\n") + log_file.close() + if output: + print (str(text) + "\n") + +del exit_if_none + # Classes and Exceptions -class NonValidFile(ValueError): pass -class NonValidDir(ValueError): pass -class NonValidCommand(ValueError): pass +class NonValidFile(ValueError): + def __init__(self): + ValueError.__init__(self) + printf(self.message) +class NonValidDir(ValueError): + def __init__(self): + ValueError.__init__(self) + printf(self.message) +class NonValidCommand(ValueError): + def __init__(self): + ValueError.__init__(self) + printf(self.message) class Package: """ An object that has information about a package. """ diff --git a/config.sh b/config.sh index 75d4b73..9a44f50 100755 --- a/config.sh +++ b/config.sh @@ -2,10 +2,6 @@ # -*- coding: utf-8 -*- source local_config -# Rsync commands -rsync_update_command="rsync -av --delay-updates --exclude='*.{abs|db}.tar.*' " -rsync_post_command="rsync -av --delete --exclude='*.abs.tar.*' " - function run_python_cmd { env \ mirror=${mirror} \ @@ -13,7 +9,6 @@ function run_python_cmd { logname=${logname} \ tempdir=${tempdir} \ archdb=${archdb} \ - docs_dir=${docs_dir} \ repodir=${repodir} \ blacklist=${blacklist} \ whitelist=${whitelist} \ diff --git a/cptobin.sh b/cptobin.sh index e803132..068d765 100755 --- a/cptobin.sh +++ b/cptobin.sh @@ -1,2 +1,2 @@ #!/bin/bash -cp -f clean_repo.py config.py config.sh get_license.sh main.sh mkpending.py filter.py ~/usr/bin/ \ No newline at end of file +cp -f clean_repo.py config.py config.sh get_license.sh main.sh mkpending.py filter.py local_config ~/usr/bin/ \ No newline at end of file diff --git a/filter.py b/filter.py index 78ad410..48e2d93 100755 --- a/filter.py +++ b/filter.py @@ -4,12 +4,17 @@ from glob import glob from repm.config import * from repm.pato2 import * -def listado(filename): +def listado(filename,start=0,end=None): """Obtiene una lista de paquetes de un archivo.""" - archivo = open(filename,"r") - lista = archivo.read().split("\n") - archivo.close() - return [pkg.split(":")[0].rstrip() for pkg in lista if pkg] + fsock = open(filename,"r") + lista = fsock.read().split("\n") + fsock.close() + if end is not None: + return [pkg.split(":")[start:end].rstrip() + for pkg in lista if pkg] + else: + return [pkg.split(":")[start].rstrip() + for pkg in lista if pkg] def pkginfo_from_filename(filename): """ Generates a Package object with info from a filename, @@ -135,29 +140,26 @@ def pkginfo_from_db(path_to_db): package_list=list() if not os.path.isfile(path_to_db): - raise NonValidFile(path_to_db + "is not a file") + raise NonValidFile(path_to_db + " is not a file") - check_output("mkdir -p " + config["archdb"]) + check_output(["mkdir", "-p", config["archdb"]]) try: db_open_tar = tarfile.open(path_to_db, 'r:gz') except tarfile.ReadError: - printf("No valid db_file %s or not readable" % path_to_db) - return(tuple()) - else: - printf("No db_file %s" % path_to_db) + raise NonValidFile("No valid db_file %s or not readable" % path_to_db) return(tuple()) for file in db_open_tar.getmembers(): - db_open_tar.extract(file, archdb) + db_open_tar.extract(file, config["archdb"]) db_open_tar.close() # Get info from file - for dir_ in glob(archdb + "/*"): + for dir_ in glob(config["archdb"] + "/*"): if isdir(dir_) and isfile(dir_ + "/desc"): package_list.append(pkginfo_from_desc( os.path.join(dir_,"desc"))) - check_output("rm -r %s/*" % archdb) - if verbose_: + check_output(["rm", "-r", config["archdb"]]) + if config["debug"]: printf(package_list) return package_list diff --git a/local_config.example b/local_config.example index 7edfde7..8015ee2 100644 --- a/local_config.example +++ b/local_config.example @@ -2,19 +2,19 @@ mirror="mirrors.eu.kernel.org" mirrorpath="::mirrors/archlinux" -# Directories +# Directories: they should end without / ## Optionals paraboladir=~/parabolagnulinux.org logtime=$(date -u +%Y%m%d-%H:%M) ## Must be defined -logname=${paraboladir}/${logtime}-repo-maintainer.log -tempdir=~/tmp/ +tempdir=~/tmp archdb=${tempdir}/db docs_dir=${paraboladir}/docs repodir=${paraboladir}/repo # End Directories # Files +logname=${paraboladir}/log/${logtime}-repo-maintainer.log blacklist=${docs_dir}/blacklist.txt whitelist=${docs_dir}/whitelist.txt pending=${docs_dir}/pending @@ -29,3 +29,7 @@ other="any" # Output options output="True" debug="False" + +# Rsync commands +rsync_update_command="rsync -av --delay-updates --exclude='*.{abs|db}.tar.*' " +rsync_post_command="rsync -av --delete --exclude='*.abs.tar.*' " \ No newline at end of file diff --git a/main.sh b/main.sh index 886ea38..46f2f75 100755 --- a/main.sh +++ b/main.sh @@ -20,7 +20,8 @@ mkrsexclude msg "Syncing repos without delete" # rsync_update_command does not sync db or abs ${rsync_update_command} --exclude-from=${rsync_blacklist} \ - ${mirror}${mirropath}/{$(echo ${repo_list} | tr ':' ',')} ${repodir} + ${mirror}${mirropath}/{$(echo ${repo_list} | tr ':' ','),\ + $(echo ${dir_list} | tr ':' ',')} ${repodir} msg "Syncing each repo and cleaning" msg2 "Remove pending files" diff --git a/mkpending.py b/mkpending.py index 2b255a8..6022206 100755 --- a/mkpending.py +++ b/mkpending.py @@ -1,9 +1,27 @@ #!/usr/bin/python # -*- coding: utf-8 -*- from repm.filter import * +import argparse -def make_pending(path_to_db): +def make_pending(path_to_db, repo, prefix=config["pending"]): """ Determine wich packages are pending for license auditing.""" + filename=prefix + "-" + repo + ".txt" + try: + fsock=open(filename, "rw") + if os.path.isfile(filename): + pkgs=[pkg for pkg in packages_iterable if pkg["name"] not in + listado(filename)] + fsock.write("\n".join([pkg["name"] + ":" + pkg["license"] + for pkg in pkgs]) + "\n") + except(IOError): + print("Can't read %s" % filename) + exit(1) + finally: + fsock.close() + + if "~" in path_to_db: + path_to_db=(os.path.expanduser(path_to_db)) + packages_iterable=pkginfo_from_db(path_to_db) search = tuple(listado(config["blacklist"]) + listado(config["whitelist"])) @@ -15,24 +33,11 @@ def make_pending(path_to_db): def write_pending(packages_iterable, repo, prefix=config["pending"]): """ Write a pending file with the info of the packages """ - filename=prefix + "-" + repo + ".txt" - try: - fsock=open(filename, "a") - except(IOError): - print("Can't read %s" % filename) - finally: - fsock.close() - if os.path.isfile(filename): - pkgs=[pkg for pkg in packages_iterable if pkg["name"] not in - listado(filename)] - fsock.write("\n".join([pkg["name"] + ":" + pkg["license"] - for pkg in pkgs]) + "\n") - fsock.close() if __name__ == "__main__": parser = argparse.ArgumentParser( description="Clean a repo db and packages") - parser.add_argument("-b", "--dababase", type=str, required=True, + parser.add_argument("-b", "--database", type=str, required=True, help="database to check") parser.add_argument("-r", "--repo", type=str, required=True, help="repo of database") diff --git a/pato2.py b/pato2.py index 4cdb536..6daa8b8 100644 --- a/pato2.py +++ b/pato2.py @@ -29,14 +29,6 @@ from repm.filter import * import tarfile from os.path import isdir, isfile -def printf(text,output=config["output"]): - """Guarda el texto en la variable log y puede imprimir en pantalla.""" - log_file = open(config["logname"], 'a') - log_file.write("\n" + str(text) + "\n") - log_file.close() - if output_: - print (str(text) + "\n") - def generate_rsync_command(base_command, dir_list=(config["repo_list"] + config["dir_list"]), @@ -68,4 +60,4 @@ def run_rsync(command,debug=config["debug"]): """ Runs rsync and gets returns it's output """ if debug: printf("rsync_command: " + command) - return check_output(command) + return check_output(command.split()) -- cgit v1.2.3-2-g168b From aca64420db93c1d1ab26ad27640458d3a1f09f3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Wed, 13 Apr 2011 00:58:19 -0500 Subject: cptobin.sh not copy inexistent files --- cptobin.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cptobin.sh b/cptobin.sh index 068d765..ce4b3bd 100755 --- a/cptobin.sh +++ b/cptobin.sh @@ -1,2 +1,2 @@ #!/bin/bash -cp -f clean_repo.py config.py config.sh get_license.sh main.sh mkpending.py filter.py local_config ~/usr/bin/ \ No newline at end of file +cp -f clean_repo.py config.py config.sh get_license.sh main.sh filter.py local_config ~/usr/bin/ \ No newline at end of file -- cgit v1.2.3-2-g168b From ddacb9a98b90fc2b51551fb73ab8513634f4f185 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Wed, 13 Apr 2011 01:01:57 -0500 Subject: filter.py to close correct file --- filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filter.py b/filter.py index 1d70a63..c7f3f18 100755 --- a/filter.py +++ b/filter.py @@ -147,7 +147,7 @@ def pkginfo_from_db(path_to_db): % path_to_db) return(tuple()) finally: - db_open_tar.close() + dbsock.close() return package_list def rsyncBlacklist_from_blacklist(packages_iterable, -- cgit v1.2.3-2-g168b From 062b7a2d9b0347e4e867e588826f052d26a3bb30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Wed, 13 Apr 2011 01:11:20 -0500 Subject: For testing --- filter.py | 2 +- local_config.example | 4 ++-- main.sh | 6 ++++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/filter.py b/filter.py index c7f3f18..4b9603d 100755 --- a/filter.py +++ b/filter.py @@ -137,7 +137,7 @@ def pkginfo_from_db(path_to_db): try: dbsock = tarfile.open(path_to_db, 'r:gz') - desc_files=[desc for desc in db_open_tar.getnames() + desc_files=[desc for desc in dbsock.getnames() if "/desc" in desc] for name in desc_files: desc=dbsock.extractfile(name) diff --git a/local_config.example b/local_config.example index 8015ee2..99a1fdb 100644 --- a/local_config.example +++ b/local_config.example @@ -31,5 +31,5 @@ output="True" debug="False" # Rsync commands -rsync_update_command="rsync -av --delay-updates --exclude='*.{abs|db}.tar.*' " -rsync_post_command="rsync -av --delete --exclude='*.abs.tar.*' " \ No newline at end of file +rsync_update_command="rsync -nav --delay-updates --exclude='*.{abs|db}.tar.*' " +rsync_post_command="rsync -nav --delete --exclude='*.abs.tar.*' " \ No newline at end of file diff --git a/main.sh b/main.sh index 9f41a95..d15f736 100755 --- a/main.sh +++ b/main.sh @@ -29,8 +29,10 @@ stdnull "rm -rf ${pending}*" for repo in $(echo ${repo_list} | tr ':' ' '); do for arch in $(echo ${arch_list} | tr ':' ' '); do msg2 "Syncing ${repo} ${arch}" - ${rsync_post_command} --exclude-from=${rsync_blacklist} \ - ${mirror}${mirropath}/${repo} ${repodir}/${repo} + cmd=$(echo ${rsync_post_command} --exclude-from=${rsync_blacklist} \ + ${mirror}${mirropath}/${repo} ${repodir}/${repo}) + plain "${cmd}" + ${cmd} msg2 "Cleaning ${repo} ${arch}" # This also generates pending lists run_python_cmd "clean_repo.py -b ${repodir}/${repo}/os/${arch}/${repo}.db.tar.gz -d ${repodir}/${repo}/os/${arch}/" -- cgit v1.2.3-2-g168b From deab65fad4ced009fb31f7033b1db8ef0af78aee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 15 Apr 2011 04:53:27 -0500 Subject: Python parts ready for bash usage in python 2 and 3 --- clean_repo.py | 78 ++++++++++++++++++++++++++++++----------------------- config.py | 65 +++++++++----------------------------------- filter.py | 48 ++++++++++++++++++++------------- pato2.py | 63 ------------------------------------------- test/test_filter.py | 3 ++- 5 files changed, 88 insertions(+), 169 deletions(-) delete mode 100644 pato2.py diff --git a/clean_repo.py b/clean_repo.py index eccfd01..5423c4d 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -3,21 +3,16 @@ from repm.filter import * import argparse -def mkpending(path_to_db, repo, prefix=config["pending"]): +def mkpending(packages_iterable, pending_file, blacklisted_names, + whitelisted_names): """ Determine wich packages are pending for license auditing.""" - if "~" in path_to_db: - path_to_db=(os.path.expanduser(path_to_db)) - - search = tuple(listado(config["blacklist"]) + - listado(config["whitelist"])) + search = tuple(blacklisted_names + + whitelisted_names) - pkgs=list(pkginfo_from_db(path_to_db)) - - filename=prefix + "-" + repo + ".txt" try: - fsock=open(filename, "rw") - pkgs=[pkg for pkg in pkginfo_from_db(path_to_db) - if pkg["name"] not in listado(filename)] + fsock=open(pending_file, "r") + pkgs=[pkg for pkg in packages_iterable + if pkg["name"] not in listado(pending_file)] for line in fsock.readlines(): if line: pkg=Package() @@ -26,16 +21,16 @@ def mkpending(path_to_db, repo, prefix=config["pending"]): pkgs.append(pkg) pkgs=[pkg for pkg in pkgs if pkg["name"] not in search and "custom" in pkg["license"]] + fsock=open(pending_file, "w") fsock.write("\n".join([pkg["name"] + ":" + pkg["license"] for pkg in pkgs]) + "\n") except(IOError): - raise NonValidFile("Can't read or write %s" % filename) + raise NonValidFile("Can't read or write %s" % pending_file) finally: fsock.close() return pkgs -def remove_from_blacklist(path_to_db, blacklisted_names, - debug=config["debug"]): +def remove_from_blacklist(path_to_db, blacklisted_names): """ Check the blacklist and remove packages on the db""" if "~" in path_to_db: path_to_db=(os.path.expanduser(path_to_db)) @@ -47,9 +42,7 @@ def remove_from_blacklist(path_to_db, blacklisted_names, cmd = "repo-remove " + path_to_db + " " + lista printf(cmd) a = check_output(cmd) - if debug: - printf(a) - return pkgs, cmd + return pkgs def cleanup_nonfree_in_dir(directory, blacklisted_names): if "~" in directory: @@ -61,25 +54,42 @@ def cleanup_nonfree_in_dir(directory, blacklisted_names): if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Clean a repo db and packages") - parser.add_argument("-b", "--database", type=str, - help="dabatase to clean") - parser.add_argument("-d", "--directory", type=str, - help="directory to clean") + prog="clean_repo", + description="Clean a repo db and packages",) + + parser.add_argument("-k", "--blacklist-file", type=str, + help="File containing blacklisted names", + required=True,) + + group_dir=parser.add_argument_group("Clean non-free packages in dir") + group_dir.add_argument("-d", "--directory", type=str, + help="directory to clean",) + + group_db=parser.add_argument_group("Clean non-free packages in db", + "All arguments need to be specified") + group_db.add_argument("-b", "--database", type=str, + help="dabatase to clean") + group_db.add_argument("-p", "--pending-file", type=str, + help="File in which to write pending list") + group_db.add_argument("-w", "--whitelist-file", type=str, + help="File containing whitelisted names") + args=parser.parse_args() + if not args.directory and not args.database: + parser.print_help() + elif not args.pending_file or not args.whitelist_file \ + and args.database: + parser.print_help() + else: + blacklisted=listado(args.blacklist_file) + if args.database: - repo=os.path.basename(args.database).split(".")[0] + whitelisted=listado(args.whitelist_file) pkgs=pkginfo_from_db(args.database) - remove_from_blacklist(args.database, pkgs, - tuple(listado(config["blacklist"]) + - listado(config["pending"] + - "-" + repo + ".txt"))) - mkpending(args.database, args.repo) + remove_from_blacklist(args.database, blacklisted) + mkpending(pkgs, args.pending_file, + blacklisted, whitelisted) if args.directory: - cleanup_nonfree_in_dir(args.directory, - listado(config["blacklist"])) - - if not args.directory and not args.database: - parser.print_help() + cleanup_nonfree_in_dir(args.directory, blacklisted) diff --git a/config.py b/config.py index 8b48c66..4e218a5 100755 --- a/config.py +++ b/config.py @@ -9,61 +9,26 @@ except(ImportError): return getoutput(cmd) import os -stringvars=("mirror", "mirrorpath", "logname", "tempdir", "archdb", - "repodir", "blacklist", "whitelist", "pending", - "rsync_blacklist",) -listvars=("repo_list", "dir_list", "arch_list", "other",) -boolvars=("output", "debug",) - -config=dict() - -def exit_if_none(var): - if os.environ.get(var) is None: - exit("%s is not defined" % var) - -for var in stringvars: - exit_if_none(var) - config[var]=os.environ.get(var) - -for var in listvars: - exit_if_none(var) - config[var]=tuple(os.environ.get(var).split(":")) - -for var in boolvars: - exit_if_none(var) - if os.environ.get(var) == "True": - config[var]=True - elif os.environ.get(var) =="False": - config[var]=False - else: - print('%s is not True or False' % var) # Rsync commands -rsync_list_command="rsync -a --no-motd --list-only " -def printf(text,output=config["output"]): +def printf(text, logfile=False): """Guarda el texto en la variable log y puede imprimir en pantalla.""" - log_file = open(config["logname"], 'a') - log_file.write("\n" + str(text) + "\n") - log_file.close() - if output: - print (str(text) + "\n") + print (str(text) + "\n") + if logfile: + try: + log = open(logfile, 'a') + log.write("\n" + str(text) + "\n") + except: + print("Can't open %s" % logfile) + finally: + log.close() -del exit_if_none # Classes and Exceptions -class NonValidFile(ValueError): - def __init__(self): - ValueError.__init__(self) - printf(self.message) -class NonValidDir(ValueError): - def __init__(self): - ValueError.__init__(self) - printf(self.message) -class NonValidCommand(ValueError): - def __init__(self): - ValueError.__init__(self) - printf(self.message) +class NonValidFile(ValueError): pass +class NonValidDir(ValueError): pass +class NonValidCommand(ValueError): pass class Package: """ An object that has information about a package. """ @@ -101,7 +66,3 @@ class Package: return False else: return True - -if __name__=="__main__": - for key in config.keys(): - print("%s : %s" % (key,config[key])) diff --git a/filter.py b/filter.py index 4b9603d..add0235 100755 --- a/filter.py +++ b/filter.py @@ -2,7 +2,7 @@ #-*- encoding: utf-8 -*- from glob import glob from repm.config import * -from repm.pato2 import * +import tarfile def listado(filename,start=0,end=None): """Obtiene una lista de paquetes de un archivo.""" @@ -28,7 +28,7 @@ def pkginfo_from_filename(filename): ---------- pkg -> Package object""" if ".pkg.tar." not in filename: - raise NonValidFile + raise NonValidFile("File is not a pacman package") pkg = Package() pkg["location"] = filename fileattrs = os.path.basename(filename).split("-") @@ -140,19 +140,18 @@ def pkginfo_from_db(path_to_db): desc_files=[desc for desc in dbsock.getnames() if "/desc" in desc] for name in desc_files: - desc=dbsock.extractfile(name) - package_list.append(pkginfo_from_desc(desc.read())) + desc=dbsock.extractfile(name).read().decode("UTF-8") + package_list.append(pkginfo_from_desc(desc)) except tarfile.ReadError: raise NonValidFile("No valid db_file %s or not readable" % path_to_db) - return(tuple()) finally: dbsock.close() return package_list def rsyncBlacklist_from_blacklist(packages_iterable, blacklisted_names, - exclude_file=config["rsync_blacklist"]): + exclude_file): """ Generate an exclude list for rsync Parameters: @@ -168,20 +167,31 @@ def rsyncBlacklist_from_blacklist(packages_iterable, pkgs=[pkg["location"] for pkg in packages_iterable if isinstance(pkg, Package) and pkg["name"] in blacklisted_names] - - try: - fsock = open(exclude_file,"w") - fsock.write("\n".join(pkgs) + "\n") - except IOError: - printf("%s wasnt written" % exclude_file) - exit(1) - finally: - fsock.close() + if exclude_file: + try: + fsock = open(exclude_file,"w") + fsock.write("\n".join(pkgs) + "\n") + except IOError: + printf("%s wasnt written" % exclude_file) + exit(1) + finally: + fsock.close() return pkgs if __name__ == "__main__": - cmd=generate_rsync_command(rsync_list_command) - a=run_rsync(cmd) - packages=pkginfo_from_rsync_output(a) - rsyncBlaclist_from_blacklist(packages,listado(blacklist)) + import argparse + parser=argparse.ArgumentParser() + parser.add_argument("-r", "--rsync-exclude-file", type=str, + help="File in which to generate exclude list", + required=True,) + parser.add_argument("-k", "--blacklist-file", type=str, + help="File containing blacklisted names", + required=True,) + parser.add_argument("-c", "--rsync-command", type=str, + help="This command will be run to get a pkg list") + args=parser.parse_args() + rsout=check_output(args.rsync_command) + packages=pkginfo_from_rsync_output(rsout) + rsyncBlaclist_from_blacklist(packages, listado(args.blacklist_file), + args.rsync_exclude_file) diff --git a/pato2.py b/pato2.py deleted file mode 100644 index 6daa8b8..0000000 --- a/pato2.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -""" - parabola.py - Copyright 2009 Rafik Mas'ad - Copyright 2010 Joshua Ismael Haase Hernández - - ---------- GNU General Public License 3 ---------- - - This file is part of Parabola. - - Parabola is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - Parabola is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with Parabola. If not, see . - - -""" -from repm.config import * -from repm.filter import * -import tarfile -from os.path import isdir, isfile - -def generate_rsync_command(base_command, - dir_list=(config["repo_list"] + - config["dir_list"]), - destdir=config["repodir"], - source=config["mirror"] +config["mirrorpath"]): - """ Generates an rsync command for executing - it by combining all parameters. - - Parameters: - ---------- - base_command -> str - dir_list -> list or tuple - destdir -> str Path to dir, dir must exist. - source -> str The source for rsync - blacklist_file -> False or str Path to file, file must exist. - - Return: - ---------- - rsync_command -> str """ - if not os.path.isdir(destdir): - print(destdir + " is not a directory") - raise NonValidDir - - dir_list="{" + ",".join(dir_list) + "}" - return " ".join((base_command, os.path.join(source, dir_list), - destdir)) - -def run_rsync(command,debug=config["debug"]): - """ Runs rsync and gets returns it's output """ - if debug: - printf("rsync_command: " + command) - return check_output(command.split()) diff --git a/test/test_filter.py b/test/test_filter.py index b6d5766..d8006f9 100644 --- a/test/test_filter.py +++ b/test/test_filter.py @@ -143,7 +143,8 @@ class generateRsyncBlacklist(unittest.TestCase): def testExcludeFiles(self): a=rsyncBlacklist_from_blacklist(self.example_package_list, - listado("blacklist_sample")) + listado("blacklist_sample"), + False) b=[self.example_package_list[0]["location"],self.example_package_list[2]["location"]] self.assertEqual(a,b) -- cgit v1.2.3-2-g168b From 374c8e2a5183cdbaefe9f54184603ad8d09e30c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sat, 16 Apr 2011 03:15:01 -0500 Subject: * bash-repo for testing * merged dbscripts with repm --- changelog | 2 -- clean_repo.py | 3 ++- config | 2 +- config.sh | 26 ---------------------- cptobin.sh | 2 -- filter.py | 5 +++-- get_license.sh | 50 +++++++++++++++++------------------------- local_config.example | 18 ++++++--------- main.sh | 62 +++++++++++++++++++++++----------------------------- 9 files changed, 60 insertions(+), 110 deletions(-) delete mode 100644 changelog delete mode 100755 config.sh delete mode 100755 cptobin.sh diff --git a/changelog b/changelog deleted file mode 100644 index 0e89f2f..0000000 --- a/changelog +++ /dev/null @@ -1,2 +0,0 @@ -repo-maintainer 1.0 - * Known working version diff --git a/clean_repo.py b/clean_repo.py index 5423c4d..e1a17c2 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -22,7 +22,8 @@ def mkpending(packages_iterable, pending_file, blacklisted_names, pkgs=[pkg for pkg in pkgs if pkg["name"] not in search and "custom" in pkg["license"]] fsock=open(pending_file, "w") - fsock.write("\n".join([pkg["name"] + ":" + pkg["license"] + fsock.write("\n".join([pkg["name"] + ":" + pkg["location"] + + ":" + pkg["license"] for pkg in pkgs]) + "\n") except(IOError): raise NonValidFile("Can't read or write %s" % pending_file) diff --git a/config b/config index 217b627..d9bad13 100644 --- a/config +++ b/config @@ -1,7 +1,7 @@ FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" -PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing') +PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing' 'multilib') PKGPOOL='pool/packages' SRCPOOL='sources/packages' diff --git a/config.sh b/config.sh deleted file mode 100755 index 9a44f50..0000000 --- a/config.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -# -*- coding: utf-8 -*- -source local_config - -function run_python_cmd { - env \ - mirror=${mirror} \ - mirrorpath=${mirrorpath} \ - logname=${logname} \ - tempdir=${tempdir} \ - archdb=${archdb} \ - repodir=${repodir} \ - blacklist=${blacklist} \ - whitelist=${whitelist} \ - pending=${pending} \ - rsync_blacklist=${rsync_blacklist} \ - repo_list=${repo_list} \ - dir_list=${dir_list} \ - arch_list=${arch_list} \ - other=${other} \ - output=${output} \ - debug=${debug} \ - $1 -} - -source libremessages \ No newline at end of file diff --git a/cptobin.sh b/cptobin.sh deleted file mode 100755 index ce4b3bd..0000000 --- a/cptobin.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -cp -f clean_repo.py config.py config.sh get_license.sh main.sh filter.py local_config ~/usr/bin/ \ No newline at end of file diff --git a/filter.py b/filter.py index add0235..4abdf38 100755 --- a/filter.py +++ b/filter.py @@ -189,9 +189,10 @@ if __name__ == "__main__": help="File containing blacklisted names", required=True,) parser.add_argument("-c", "--rsync-command", type=str, - help="This command will be run to get a pkg list") + help="This command will be run to get a pkg list", + required=True,) args=parser.parse_args() - rsout=check_output(args.rsync_command) + rsout=check_output(args.rsync_command.split()) packages=pkginfo_from_rsync_output(rsout) rsyncBlaclist_from_blacklist(packages, listado(args.blacklist_file), args.rsync_exclude_file) diff --git a/get_license.sh b/get_license.sh index 0da58cb..024876c 100755 --- a/get_license.sh +++ b/get_license.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # -*- coding: utf-8 -*- # get_license.sh @@ -20,36 +20,26 @@ # You should have received a copy of the GNU General Public License # along with Parabola. If not, see . - -docs="/home/parabolavnx/parabolagnulinux.org/docs" -repo="/home/parabolavnx/parabolagnulinux.org/repo" -dir="$docs/pending-licenses" - -echo "Cleaning $dir" -rm -rf $dir/* - -tempdir=$(mktemp -d) -cd $tempdir - -pending=($(cut -d: -f1 $docs/pending*.txt)) -echo ${pending[@]} - -for pkg in ${pending[@]}; do - pkg_in_repo=( $(ls ${repo}/*/os/*/${pkg}*) ) - for y in ${pkg_in_repo[@]}; do - echo "chmod +r $y" - chmod +r $y - echo "tar -xf $y usr/share/licenses" - bsdtar -xf $y usr/share/licenses - echo "chmod -r $y" - chmod -r $y +source ./config +source ./local_config +source ./libremessages + +msg "Creating pending licenses list" +pushd ${licenses_dir} +rm -rf ${licenses_dir}/* + +for repo in ${PKGREPOS[@]}; do + msg2 "Extracting licenses in ${repo}" + pending=($(cut -d: -f2 ${docs_dir}/pending-${repo})) + pushd ${repodir}/${repo} + for pkg in ${pending[@]}; do + plain "${pkg}" + bsdtar -xf ${pkg} usr/share/licenses || { + error "${pkg} has no licenses" + } + chmod -r ${pkg} done done -mv usr/share/licenses/* $dir - -cd - -rm -rf $tempdir - +popd exit 0 \ No newline at end of file diff --git a/local_config.example b/local_config.example index 99a1fdb..2c0ef3e 100644 --- a/local_config.example +++ b/local_config.example @@ -4,21 +4,21 @@ mirrorpath="::mirrors/archlinux" # Directories: they should end without / ## Optionals -paraboladir=~/parabolagnulinux.org -logtime=$(date -u +%Y%m%d-%H:%M) +paraboladir= ## Must be defined tempdir=~/tmp archdb=${tempdir}/db docs_dir=${paraboladir}/docs -repodir=${paraboladir}/repo +repodir=${paraboladir}/repo/staging +licenses_dir=${docs_dir}/pending_licenses # End Directories # Files -logname=${paraboladir}/log/${logtime}-repo-maintainer.log +logname=${paraboladir}/log/$(date -u +%Y%m%d-%H:%M)-repo-maintainer.log blacklist=${docs_dir}/blacklist.txt whitelist=${docs_dir}/whitelist.txt -pending=${docs_dir}/pending rsync_blacklist=${docs_dir}/rsyncBlacklist +rsync_not_needed=${tmp}/rsync_not_needed # Repos, arches, and dirs for repo repo_list="core:extra:community:testing:community-testing:multilib" @@ -26,10 +26,6 @@ dir_list="pool" arch_list="i686:x86_64" other="any" -# Output options -output="True" -debug="False" - # Rsync commands -rsync_update_command="rsync -nav --delay-updates --exclude='*.{abs|db}.tar.*' " -rsync_post_command="rsync -nav --delete --exclude='*.abs.tar.*' " \ No newline at end of file +rsync_list_command="rsync\ -ptgoL\ --list-only\ " +rsync_update_command="rsync -ptgoL --exclude='*.abs.tar.*'" diff --git a/main.sh b/main.sh index d15f736..b3ecb92 100755 --- a/main.sh +++ b/main.sh @@ -1,43 +1,35 @@ #!/bin/bash # -*- coding: utf-8 -*- -source config.sh +source ./config +source ./local_config +source ./libremessages -function mkrsexclude { - local error=1 - while ${error}; do - run_python_cmd "filter.py" - error=$? - done -} - -msg "Cleaning $tempdir" -stdnull "rm -r $tempdir/* " - -msg "Generating exclude list for rsync" -mkrsexclude - -msg "Syncing repos without delete" -# rsync_update_command does not sync db or abs -${rsync_update_command} --exclude-from=${rsync_blacklist} \ - ${mirror}${mirropath}/{$(echo ${repo_list} | tr ':' ','),\ - $(echo ${dir_list} | tr ':' ',')} ${repodir} - -msg "Syncing each repo and cleaning" -msg2 "Remove pending files" -stdnull "rm -rf ${pending}*" -for repo in $(echo ${repo_list} | tr ':' ' '); do - for arch in $(echo ${arch_list} | tr ':' ' '); do - msg2 "Syncing ${repo} ${arch}" - cmd=$(echo ${rsync_post_command} --exclude-from=${rsync_blacklist} \ - ${mirror}${mirropath}/${repo} ${repodir}/${repo}) - plain "${cmd}" - ${cmd} - msg2 "Cleaning ${repo} ${arch}" - # This also generates pending lists - run_python_cmd "clean_repo.py -b ${repodir}/${repo}/os/${arch}/${repo}.db.tar.gz -d ${repodir}/${repo}/os/${arch}/" +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]} 'any'; do + msg "Syncing ${repo} ${arch}" + filter.py -r "${rsync_blacklist}" -k "${blacklist}" -c \ + \"${rsync_list_command}\ \ + ${mirror}${mirrorpath}/${repo}/os/${arch}\ \ + ${repodir}/${repo}/\" + find ${repodir}/${repo} -name *${PKGEXT} -print \ + > ${rsync_not_needed} + ${rsync_update_command} \ + ${mirror}${mirrorpath}/${repo}/os/${arch} \ + ${repodir}/${repo} \ + --exclude-from=${rsync_blacklist} \ + --exclude-from=${rsync_not_needed} done + for arch in ${ARCHES[@]}; do + if [ -r ${repodir}/${repo}/os/${arch}/${repo}${DBEXT} ]; then + clean_repo.py -k ${blacklist} -w ${whitelist} \ + -p ${docs_dir}/pending-${repo} \ + -b ${repodir}/${repo}/${repo}${DBEXT} + fi + clean_repo.py -k ${blacklist} -d ${repodir}/${repo} done -msg "Checking licenses" +db-update +ftpdir-cleanup + get_license.sh -- cgit v1.2.3-2-g168b From bbb2155b1bb0dd442be86e05d1be6400e62be5d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sat, 16 Apr 2011 03:40:54 -0500 Subject: Updated TODO list. --- TODO | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/TODO b/TODO index 11ef51a..e50cefa 100644 --- a/TODO +++ b/TODO @@ -1,6 +1,10 @@ -* Test Suite +* Test Suite for clean_repo.py - - Review all repo efectively - - Remove all blacklisted packages - - Get pending list right - - Extract licenses all right \ No newline at end of file + - Review all repo + - Remove all blacklisted packages + - Get pending list right + - Extract licenses all right + +* Fix db-move + + - Make it use abslibre \ No newline at end of file -- cgit v1.2.3-2-g168b From bef1e554380b797a4dfff5950c5aea59d093b4be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sat, 16 Apr 2011 12:16:21 -0500 Subject: main.sh renamed to repo-update --- repo-state | 11 ----------- repo-update | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 11 deletions(-) delete mode 100755 repo-state create mode 100755 repo-update diff --git a/repo-state b/repo-state deleted file mode 100755 index 319db83..0000000 --- a/repo-state +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh - -a=$(date +%Y%m%d+%H%M) -mkdir -p $a -cd $a - -. /home/joshpar/programas/arch2parabola/repo-list-diff -# scp parabolavnx@parabolagnulinux.org:~/tmp/rsyncBlacklist.txt ./ - -cd .. -exit 0 \ No newline at end of file diff --git a/repo-update b/repo-update new file mode 100755 index 0000000..b3ecb92 --- /dev/null +++ b/repo-update @@ -0,0 +1,35 @@ +#!/bin/bash +# -*- coding: utf-8 -*- + +source ./config +source ./local_config +source ./libremessages + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]} 'any'; do + msg "Syncing ${repo} ${arch}" + filter.py -r "${rsync_blacklist}" -k "${blacklist}" -c \ + \"${rsync_list_command}\ \ + ${mirror}${mirrorpath}/${repo}/os/${arch}\ \ + ${repodir}/${repo}/\" + find ${repodir}/${repo} -name *${PKGEXT} -print \ + > ${rsync_not_needed} + ${rsync_update_command} \ + ${mirror}${mirrorpath}/${repo}/os/${arch} \ + ${repodir}/${repo} \ + --exclude-from=${rsync_blacklist} \ + --exclude-from=${rsync_not_needed} + done + for arch in ${ARCHES[@]}; do + if [ -r ${repodir}/${repo}/os/${arch}/${repo}${DBEXT} ]; then + clean_repo.py -k ${blacklist} -w ${whitelist} \ + -p ${docs_dir}/pending-${repo} \ + -b ${repodir}/${repo}/${repo}${DBEXT} + fi + clean_repo.py -k ${blacklist} -d ${repodir}/${repo} +done + +db-update +ftpdir-cleanup + +get_license.sh -- cgit v1.2.3-2-g168b From fbc8eec9e5cb3d598fb84579811f16ff8c2b71d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sat, 16 Apr 2011 12:26:13 -0500 Subject: Fixed error in repo-update --- repo-update | 1 + 1 file changed, 1 insertion(+) diff --git a/repo-update b/repo-update index b3ecb92..59a0658 100755 --- a/repo-update +++ b/repo-update @@ -27,6 +27,7 @@ for repo in ${PKGREPOS[@]}; do -b ${repodir}/${repo}/${repo}${DBEXT} fi clean_repo.py -k ${blacklist} -d ${repodir}/${repo} + done done db-update -- cgit v1.2.3-2-g168b From e0aad034cd3b7e91137f85584f9b53cdc871f44f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sat, 16 Apr 2011 12:42:00 -0500 Subject: Modified local_config.example --- filter.py | 4 ++-- local_config.example | 10 +--------- main.sh | 35 ----------------------------------- 3 files changed, 3 insertions(+), 46 deletions(-) delete mode 100755 main.sh diff --git a/filter.py b/filter.py index 4abdf38..c71f54d 100755 --- a/filter.py +++ b/filter.py @@ -4,9 +4,9 @@ from glob import glob from repm.config import * import tarfile -def listado(filename,start=0,end=None): +def listado(filename, start=0, end=None): """Obtiene una lista de paquetes de un archivo.""" - fsock = open(filename,"r") + fsock = open(filename, "r") lista = fsock.read().split("\n") fsock.close() if end is not None: diff --git a/local_config.example b/local_config.example index 2c0ef3e..0a827f4 100644 --- a/local_config.example +++ b/local_config.example @@ -3,9 +3,7 @@ mirror="mirrors.eu.kernel.org" mirrorpath="::mirrors/archlinux" # Directories: they should end without / -## Optionals -paraboladir= -## Must be defined +paraboladir=~/parabolagnulinux.org tempdir=~/tmp archdb=${tempdir}/db docs_dir=${paraboladir}/docs @@ -20,12 +18,6 @@ whitelist=${docs_dir}/whitelist.txt rsync_blacklist=${docs_dir}/rsyncBlacklist rsync_not_needed=${tmp}/rsync_not_needed -# Repos, arches, and dirs for repo -repo_list="core:extra:community:testing:community-testing:multilib" -dir_list="pool" -arch_list="i686:x86_64" -other="any" - # Rsync commands rsync_list_command="rsync\ -ptgoL\ --list-only\ " rsync_update_command="rsync -ptgoL --exclude='*.abs.tar.*'" diff --git a/main.sh b/main.sh deleted file mode 100755 index b3ecb92..0000000 --- a/main.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# -*- coding: utf-8 -*- - -source ./config -source ./local_config -source ./libremessages - -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]} 'any'; do - msg "Syncing ${repo} ${arch}" - filter.py -r "${rsync_blacklist}" -k "${blacklist}" -c \ - \"${rsync_list_command}\ \ - ${mirror}${mirrorpath}/${repo}/os/${arch}\ \ - ${repodir}/${repo}/\" - find ${repodir}/${repo} -name *${PKGEXT} -print \ - > ${rsync_not_needed} - ${rsync_update_command} \ - ${mirror}${mirrorpath}/${repo}/os/${arch} \ - ${repodir}/${repo} \ - --exclude-from=${rsync_blacklist} \ - --exclude-from=${rsync_not_needed} - done - for arch in ${ARCHES[@]}; do - if [ -r ${repodir}/${repo}/os/${arch}/${repo}${DBEXT} ]; then - clean_repo.py -k ${blacklist} -w ${whitelist} \ - -p ${docs_dir}/pending-${repo} \ - -b ${repodir}/${repo}/${repo}${DBEXT} - fi - clean_repo.py -k ${blacklist} -d ${repodir}/${repo} -done - -db-update -ftpdir-cleanup - -get_license.sh -- cgit v1.2.3-2-g168b From e341449526be63b061e6d2661e5f539b959e8c07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sat, 16 Apr 2011 14:49:51 -0500 Subject: Will this solve problems? --- repo-update | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/repo-update b/repo-update index 59a0658..495f515 100755 --- a/repo-update +++ b/repo-update @@ -8,7 +8,7 @@ source ./libremessages for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]} 'any'; do msg "Syncing ${repo} ${arch}" - filter.py -r "${rsync_blacklist}" -k "${blacklist}" -c \ + python filter.py -r "${rsync_blacklist}" -k "${blacklist}" -c \ \"${rsync_list_command}\ \ ${mirror}${mirrorpath}/${repo}/os/${arch}\ \ ${repodir}/${repo}/\" @@ -22,11 +22,11 @@ for repo in ${PKGREPOS[@]}; do done for arch in ${ARCHES[@]}; do if [ -r ${repodir}/${repo}/os/${arch}/${repo}${DBEXT} ]; then - clean_repo.py -k ${blacklist} -w ${whitelist} \ + python clean_repo.py -k ${blacklist} -w ${whitelist} \ -p ${docs_dir}/pending-${repo} \ -b ${repodir}/${repo}/${repo}${DBEXT} fi - clean_repo.py -k ${blacklist} -d ${repodir}/${repo} + python clean_repo.py -k ${blacklist} -d ${repodir}/${repo} done done -- cgit v1.2.3-2-g168b From 6ce3ddee58415e052ec578e777994a60c7e4c3a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sun, 17 Apr 2011 03:11:36 -0300 Subject: sourceballs2 --- cron-jobs/sourceballs2 | 15 +++++++++++++-- db-functions | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index ba35298..b70f417 100644 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -21,8 +21,9 @@ find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort pushd "${SVNREPO}" >/dev/null -failedpkgs=() for repo in ${PKGREPOS[@]}; do + failedpkgs=() + pushd $repo >/dev/null find . -maxdepth 1 -type d | while read pkg; do pushd "${SVNREPO}/$repo/$pkg" >/dev/null @@ -42,13 +43,23 @@ for repo in ${PKGREPOS[@]}; do [[ -e "${SRCPKGDEST}/${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" ]] && \ continue + msg2 "$pkgbase-$pkgver-$pkgrel..." makepkg --allsource --ignorearch -c - [[ $? -ne 0 ]] && \ + [[ $? -ne 0 ]] && { + warning "Failed." failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" + } done popd >/dev/null + + if [ ${#failedpkgs[@]} -ge 1 ]; then + msg "Failed to create source packages for [${repo}]..." + for failed_pkg in ${failedpkgs[@]}; do + msg2 "${failed_pkg}" + done + fi done # Cleanup old source packages diff --git a/db-functions b/db-functions index e2731cb..5953ed1 100644 --- a/db-functions +++ b/db-functions @@ -353,6 +353,7 @@ check_splitpkgs() { for pkgfile in ${pkgfiles[@]}; do issplitpkg "${pkgfile}" || continue local _pkgbase="$(getpkgbase ${pkgfile})" + msg2 "Checking $_pkgbase" local _pkgname="$(getpkgname ${pkgfile})" local _pkgarch="$(getpkgarch ${pkgfile})" mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" -- cgit v1.2.3-2-g168b From 86d44994d4129c0cf0ffdc9cba71303b870f14d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sun, 17 Apr 2011 03:24:42 -0300 Subject: fixed merge --- db-functions | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/db-functions b/db-functions index 6e0a6d3..fb20e3e 100644 --- a/db-functions +++ b/db-functions @@ -497,10 +497,7 @@ arch_repo_add() { # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null -<<<<<<< HEAD - repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ -======= - /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} \ + repo-add -q "${repo}${DBEXT}" ${pkgs[@]} \ >>>>>>> 1ce0c6368d0908e25f9bd1bb8183b5f29053fac8 || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" /usr/bin/repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ @@ -520,11 +517,7 @@ arch_repo_remove() { error "No database found at '${dbfile}'" return 1 fi -<<<<<<< HEAD - repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ -======= - /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} \ ->>>>>>> 1ce0c6368d0908e25f9bd1bb8183b5f29053fac8 + repo-remove -q "${dbfile}" ${pkgs[@]} \ || error "repo-remove ${dbfile} ${pkgs[@]}" /usr/bin/repo-remove -q "${filesfile}" ${pkgs[@]} \ || error "repo-remove ${filesfile} ${pkgs[@]}" -- cgit v1.2.3-2-g168b From e891caf97ee563e52b8f5e1f2448db2b0dc49228 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sun, 17 Apr 2011 03:26:03 -0300 Subject: More merge fixes --- db-functions | 1 - 1 file changed, 1 deletion(-) diff --git a/db-functions b/db-functions index fb20e3e..ff38d62 100644 --- a/db-functions +++ b/db-functions @@ -498,7 +498,6 @@ arch_repo_add() { # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null repo-add -q "${repo}${DBEXT}" ${pkgs[@]} \ ->>>>>>> 1ce0c6368d0908e25f9bd1bb8183b5f29053fac8 || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" /usr/bin/repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ || error "repo-add -f ${repo}${FILESEXT} ${pkgs[@]}" -- cgit v1.2.3-2-g168b From c44ba7fa5154e71921d64f2065a71a4c6755c61b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 19 Apr 2011 19:14:41 -0500 Subject: Next fix --- .gitignore | 2 ++ clean_repo.py | 12 ++++++++---- filter.py | 16 +++++++++++----- local_config.example | 13 ++++++++----- repo-update | 19 ++++++++++--------- 5 files changed, 39 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index fb6241a..45688ab 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ local_config /config.local test/packages/*/*.pkg.tar.?z +#*# +.#* \ No newline at end of file diff --git a/clean_repo.py b/clean_repo.py index e1a17c2..bc401b5 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -48,10 +48,13 @@ def remove_from_blacklist(path_to_db, blacklisted_names): def cleanup_nonfree_in_dir(directory, blacklisted_names): if "~" in directory: directory=(os.path.expanduser(directory)) + pkglist=list() pkgs=pkginfo_from_files_in_dir(directory) for package in pkgs: if package["name"] in blacklisted_names: os.remove(package["location"]) + pkglist.append(package) + return pkglist if __name__ == "__main__": parser = argparse.ArgumentParser( @@ -64,10 +67,10 @@ if __name__ == "__main__": group_dir=parser.add_argument_group("Clean non-free packages in dir") group_dir.add_argument("-d", "--directory", type=str, - help="directory to clean",) + help="directory to clean") group_db=parser.add_argument_group("Clean non-free packages in db", - "All arguments need to be specified") + "All arguments need to be specified for db cleaning") group_db.add_argument("-b", "--database", type=str, help="dabatase to clean") group_db.add_argument("-p", "--pending-file", type=str, @@ -82,8 +85,8 @@ if __name__ == "__main__": elif not args.pending_file or not args.whitelist_file \ and args.database: parser.print_help() - else: - blacklisted=listado(args.blacklist_file) + + blacklisted=listado(args.blacklist_file) if args.database: whitelisted=listado(args.whitelist_file) @@ -94,3 +97,4 @@ if __name__ == "__main__": if args.directory: cleanup_nonfree_in_dir(args.directory, blacklisted) + diff --git a/filter.py b/filter.py index c71f54d..5d90bdd 100755 --- a/filter.py +++ b/filter.py @@ -1,4 +1,4 @@ - #! /usr/bin/python +#! /usr/bin/python #-*- encoding: utf-8 -*- from glob import glob from repm.config import * @@ -188,11 +188,17 @@ if __name__ == "__main__": parser.add_argument("-k", "--blacklist-file", type=str, help="File containing blacklisted names", required=True,) - parser.add_argument("-c", "--rsync-command", type=str, - help="This command will be run to get a pkg list", + parser.add_argument("-f", "--rsout-file", type=str, + help="This file will be read to get a pkg list", required=True,) args=parser.parse_args() - rsout=check_output(args.rsync_command.split()) + try: + fsock=open(args.rsout_file, "r") + rsout=fsock.read() + except IOError: + print("%s is not readable" % args.rsout_file) + finally: + fsock.close() packages=pkginfo_from_rsync_output(rsout) - rsyncBlaclist_from_blacklist(packages, listado(args.blacklist_file), + rsyncBlacklist_from_blacklist(packages, listado(args.blacklist_file), args.rsync_exclude_file) diff --git a/local_config.example b/local_config.example index 0a827f4..05694e9 100644 --- a/local_config.example +++ b/local_config.example @@ -1,6 +1,6 @@ # Mirror options mirror="mirrors.eu.kernel.org" -mirrorpath="::mirrors/archlinux" +mirrorpath="mirrors/archlinux" # Directories: they should end without / paraboladir=~/parabolagnulinux.org @@ -13,11 +13,14 @@ licenses_dir=${docs_dir}/pending_licenses # Files logname=${paraboladir}/log/$(date -u +%Y%m%d-%H:%M)-repo-maintainer.log +rsout_file=${tempdir}/rsout +rsync_not_needed=${tempdir}/rsync_not_needed + +rsync_blacklist=${docs_dir}/rsyncBlacklist + blacklist=${docs_dir}/blacklist.txt whitelist=${docs_dir}/whitelist.txt -rsync_blacklist=${docs_dir}/rsyncBlacklist -rsync_not_needed=${tmp}/rsync_not_needed # Rsync commands -rsync_list_command="rsync\ -ptgoL\ --list-only\ " -rsync_update_command="rsync -ptgoL --exclude='*.abs.tar.*'" +rsync_list_command="rsync -rptgoL --exclude='*.abs.tar.*' --list-only --no-motd " +rsync_update_command="rsync -rptgoL --exclude='*.abs.tar.*' --no-motd " diff --git a/repo-update b/repo-update index 495f515..9b36d75 100755 --- a/repo-update +++ b/repo-update @@ -8,21 +8,22 @@ source ./libremessages for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]} 'any'; do msg "Syncing ${repo} ${arch}" - python filter.py -r "${rsync_blacklist}" -k "${blacklist}" -c \ - \"${rsync_list_command}\ \ - ${mirror}${mirrorpath}/${repo}/os/${arch}\ \ - ${repodir}/${repo}/\" - find ${repodir}/${repo} -name *${PKGEXT} -print \ - > ${rsync_not_needed} + ${rsync_list_command} \ + ${mirror}::${mirrorpath}/${repo}/os/${arch}/ \ + ${repodir}/${repo}/ > ${rsout_file} + filter.py -r ${rsync_blacklist} -k ${blacklist} \ + -f ${rsout_file} + find ${repodir}/${repo} -name *${PKGEXT} \ + -fprint ${rsync_not_needed} ${rsync_update_command} \ - ${mirror}${mirrorpath}/${repo}/os/${arch} \ - ${repodir}/${repo} \ --exclude-from=${rsync_blacklist} \ --exclude-from=${rsync_not_needed} + ${mirror}::${mirrorpath}/${repo}/os/${arch}/ \ + ${repodir}/${repo}/ done for arch in ${ARCHES[@]}; do if [ -r ${repodir}/${repo}/os/${arch}/${repo}${DBEXT} ]; then - python clean_repo.py -k ${blacklist} -w ${whitelist} \ + clean_repo.py -k ${blacklist} -w ${whitelist} \ -p ${docs_dir}/pending-${repo} \ -b ${repodir}/${repo}/${repo}${DBEXT} fi -- cgit v1.2.3-2-g168b From 79e48771c2d64b50db5c8dadb4636e98fab94480 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 19 Apr 2011 19:23:37 -0500 Subject: Next fix --- repo-update | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/repo-update b/repo-update index 9b36d75..2354e0a 100755 --- a/repo-update +++ b/repo-update @@ -1,9 +1,9 @@ #!/bin/bash # -*- coding: utf-8 -*- -source ./config -source ./local_config -source ./libremessages +source config +source local_config +source libremessages for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]} 'any'; do -- cgit v1.2.3-2-g168b From a79cd8ac55d222e182bde411b94bb19cb74fa728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 19 Apr 2011 18:36:24 -0700 Subject: rsync, filter.py, find cmd working now --- repo-update | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/repo-update b/repo-update index 2354e0a..719ab0c 100755 --- a/repo-update +++ b/repo-update @@ -6,28 +6,39 @@ source local_config source libremessages for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]} 'any'; do + for arch in 'i686' 'x86_64' 'any'; do msg "Syncing ${repo} ${arch}" + # makes a file containing rsync output for filter.py ${rsync_list_command} \ - ${mirror}::${mirrorpath}/${repo}/os/${arch}/ \ - ${repodir}/${repo}/ > ${rsout_file} + rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ + ${repodir}/staging/${repo}/ > ${rsout_file} + # reads blacklist and rsout_file and makes an rsync exclude-from + # list filter.py -r ${rsync_blacklist} -k ${blacklist} \ -f ${rsout_file} - find ${repodir}/${repo} -name *${PKGEXT} \ - -fprint ${rsync_not_needed} + # list files in ${repodir}/${repo} and write their names on + # rsync_not_needed for using as an rsync exclude-from + find ${repodir}/${repo} -name "*${PKGEXT}" \ + -fprintf ${rsync_not_needed} '%f\n' + # Actual rsync command ${rsync_update_command} \ --exclude-from=${rsync_blacklist} \ - --exclude-from=${rsync_not_needed} - ${mirror}::${mirrorpath}/${repo}/os/${arch}/ \ - ${repodir}/${repo}/ + --exclude-from=${rsync_not_needed} \ + rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ + ${repodir}/staging/${repo}/ done for arch in ${ARCHES[@]}; do + msg2 "Cleaning" + # if there is a db in repo (db is created on rsync) if [ -r ${repodir}/${repo}/os/${arch}/${repo}${DBEXT} ]; then + # clean_repo makes pending list with files on db and remove + # packages from db clean_repo.py -k ${blacklist} -w ${whitelist} \ -p ${docs_dir}/pending-${repo} \ - -b ${repodir}/${repo}/${repo}${DBEXT} + -b ${repodir}/staging/${repo}/${repo}${DBEXT} fi - python clean_repo.py -k ${blacklist} -d ${repodir}/${repo} + # if some nonfree files got pass the filter this command delete them + python clean_repo.py -k ${blacklist} -d ${repodir}/staging/${repo} done done -- cgit v1.2.3-2-g168b From b8a883720066e4f91b162ea91719f9a0f15958ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 19 Apr 2011 21:38:28 -0500 Subject: Another fix --- clean_repo.py | 6 ++---- config | 3 ++- repo-update | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/clean_repo.py b/clean_repo.py index bc401b5..6f3d632 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -80,11 +80,9 @@ if __name__ == "__main__": args=parser.parse_args() - if not args.directory and not args.database: - parser.print_help() - elif not args.pending_file or not args.whitelist_file \ - and args.database: + if args.database and not (args.pending_file and args.whitelist_file): parser.print_help() + exit(1) blacklisted=listado(args.blacklist_file) diff --git a/config b/config index 300893a..7180f9e 100644 --- a/config +++ b/config @@ -1,7 +1,8 @@ FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" -PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing' 'multilib') +ARCHREPOS=('core' 'extra' 'community' 'testing' 'multilib') +PKGREPOS=(${ARCHREPOS[@]} 'libre' 'libre-testing' 'social' 'sugar') PKGPOOL='pool/packages' SRCPOOL='sources/packages' diff --git a/repo-update b/repo-update index 719ab0c..7241507 100755 --- a/repo-update +++ b/repo-update @@ -5,7 +5,7 @@ source config source local_config source libremessages -for repo in ${PKGREPOS[@]}; do +for repo in ${ARCHREPOS[@]}; do for arch in 'i686' 'x86_64' 'any'; do msg "Syncing ${repo} ${arch}" # makes a file containing rsync output for filter.py -- cgit v1.2.3-2-g168b From ea23d5dff7b4e99e62ab582429f4a6f8b0263363 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 19 Apr 2011 21:02:56 -0700 Subject: Ready for testing in regular usage --- clean_repo.py | 9 ++++----- get_license.sh | 34 ++++++++++++++++++++++------------ local_config.example | 2 +- repo-update | 13 +++++++------ 4 files changed, 34 insertions(+), 24 deletions(-) diff --git a/clean_repo.py b/clean_repo.py index 6f3d632..51e62d4 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -25,10 +25,9 @@ def mkpending(packages_iterable, pending_file, blacklisted_names, fsock.write("\n".join([pkg["name"] + ":" + pkg["location"] + ":" + pkg["license"] for pkg in pkgs]) + "\n") - except(IOError): - raise NonValidFile("Can't read or write %s" % pending_file) - finally: fsock.close() + except(IOError): + printf("Can't read or write %s" % pending_file) return pkgs def remove_from_blacklist(path_to_db, blacklisted_names): @@ -70,7 +69,7 @@ if __name__ == "__main__": help="directory to clean") group_db=parser.add_argument_group("Clean non-free packages in db", - "All arguments need to be specified for db cleaning") + "All these arguments need to be specified for db cleaning:") group_db.add_argument("-b", "--database", type=str, help="dabatase to clean") group_db.add_argument("-p", "--pending-file", type=str, @@ -89,7 +88,7 @@ if __name__ == "__main__": if args.database: whitelisted=listado(args.whitelist_file) pkgs=pkginfo_from_db(args.database) - remove_from_blacklist(args.database, blacklisted) + # remove_from_blacklist(args.database, blacklisted) mkpending(pkgs, args.pending_file, blacklisted, whitelisted) diff --git a/get_license.sh b/get_license.sh index 024876c..b768cd5 100755 --- a/get_license.sh +++ b/get_license.sh @@ -19,27 +19,37 @@ # GNU General Public License for more details. # You should have received a copy of the GNU General Public License - # along with Parabola. If not, see . + # along with Parabola. If not, see . + source ./config source ./local_config source ./libremessages msg "Creating pending licenses list" -pushd ${licenses_dir} +pushd ${licenses_dir} >/dev/null rm -rf ${licenses_dir}/* +popd >/dev/null + +dir=$(mktemp -d ${tempdir}/licenses.XXXX) +pushd $dir > /dev/null -for repo in ${PKGREPOS[@]}; do +for repo in ${ARCHREPOS[@]}; do msg2 "Extracting licenses in ${repo}" - pending=($(cut -d: -f2 ${docs_dir}/pending-${repo})) - pushd ${repodir}/${repo} - for pkg in ${pending[@]}; do + pending=($(cut -d: -f1 ${docs_dir}/pending-${repo}.txt)) + for name in ${pending[@]}; do plain "${pkg}" - bsdtar -xf ${pkg} usr/share/licenses || { - error "${pkg} has no licenses" - } - chmod -r ${pkg} + for pkg in $(find ${repodir}/staging/${repo} -name "${name}-*${PKGEXT}" -printf '%f '); do + chmod +r ${pkg} + bsdtar -xf ${pkg} usr/share/licenses || { + error "${pkg} has no licenses" + } + chmod -r ${pkg} + done done done -popd -exit 0 \ No newline at end of file +mv ${dir}/* ${licenses_dir}/ +rm -rf ${dir} + + +exit 0 diff --git a/local_config.example b/local_config.example index 05694e9..2280cc2 100644 --- a/local_config.example +++ b/local_config.example @@ -7,7 +7,7 @@ paraboladir=~/parabolagnulinux.org tempdir=~/tmp archdb=${tempdir}/db docs_dir=${paraboladir}/docs -repodir=${paraboladir}/repo/staging +repodir=${paraboladir}/repo licenses_dir=${docs_dir}/pending_licenses # End Directories diff --git a/repo-update b/repo-update index 7241507..2133f46 100755 --- a/repo-update +++ b/repo-update @@ -27,19 +27,20 @@ for repo in ${ARCHREPOS[@]}; do rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ ${repodir}/staging/${repo}/ done - for arch in ${ARCHES[@]}; do - msg2 "Cleaning" + for arch in 'i686' 'x86_64'; do + msg "Cleaning $repo $arch" # if there is a db in repo (db is created on rsync) - if [ -r ${repodir}/${repo}/os/${arch}/${repo}${DBEXT} ]; then + if [ -r ${repodir}/staging/${repo}/os/${arch}/${repo}${DBEXT} ]; then # clean_repo makes pending list with files on db and remove # packages from db clean_repo.py -k ${blacklist} -w ${whitelist} \ - -p ${docs_dir}/pending-${repo} \ + -p ${docs_dir}/pending-${repo}.txt \ -b ${repodir}/staging/${repo}/${repo}${DBEXT} fi - # if some nonfree files got pass the filter this command delete them - python clean_repo.py -k ${blacklist} -d ${repodir}/staging/${repo} done + # if some nonfree files got pass the filter this command delete them + msg "Fallback cleaning repo" + python clean_repo.py -k ${blacklist} -d ${repodir}/staging/${repo} done db-update -- cgit v1.2.3-2-g168b From f405f3abb1ae471e36d04e688e6583c2079ecc38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Tue, 19 Apr 2011 21:09:35 -0700 Subject: using full path for ftpdir-cleanup --- repo-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo-update b/repo-update index 2133f46..697e188 100755 --- a/repo-update +++ b/repo-update @@ -44,6 +44,6 @@ for repo in ${ARCHREPOS[@]}; do done db-update -ftpdir-cleanup +~/repm/cron-jobs/ftpdir-cleanup get_license.sh -- cgit v1.2.3-2-g168b From e3325948742bc89c57836de0db330a64dcfe8afc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 22 Apr 2011 00:51:22 -0500 Subject: fix ? --- get_license.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/get_license.sh b/get_license.sh index b768cd5..1b755c6 100755 --- a/get_license.sh +++ b/get_license.sh @@ -51,5 +51,4 @@ done mv ${dir}/* ${licenses_dir}/ rm -rf ${dir} - exit 0 -- cgit v1.2.3-2-g168b From bba8c33a039ec4d1e1d13f88304ba0aeb754b03e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 22 Apr 2011 00:30:43 -0700 Subject: Midnight fix --- config | 7 +++++-- db-functions | 6 +++--- db-update | 7 ++++--- get_license.sh | 54 ------------------------------------------------------ repo-update | 17 +++++++++-------- 5 files changed, 21 insertions(+), 70 deletions(-) delete mode 100755 get_license.sh diff --git a/config b/config index 7180f9e..6936a9b 100644 --- a/config +++ b/config @@ -1,6 +1,8 @@ -FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" +#!/bin/bash +FTP_BASE="/home/parabolavnx/parabolagnulinux.org/repo" ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" + ARCHREPOS=('core' 'extra' 'community' 'testing' 'multilib') PKGREPOS=(${ARCHREPOS[@]} 'libre' 'libre-testing' 'social' 'sugar') PKGPOOL='pool/packages' @@ -23,7 +25,8 @@ LOCK_TIMEOUT=300 STAGING="$FTP_BASE/staging" TMPDIR="$HOME/tmp" -ARCHES=(i686 x86_64 mips64el) +ARCHARCHES=(i686 x86_64) +ARCHES=(${ARCHARCHES[@]} mips64el) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.?z" diff --git a/db-functions b/db-functions index 20bfa0e..4dddcb5 100644 --- a/db-functions +++ b/db-functions @@ -211,7 +211,7 @@ repo_unlock () { #repo_unlock _grep_pkginfo() { local _ret - _ret="$(bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 "^${2} = ")" + _ret="$(bsdtar -xOqf "$1" .PKGINFO | grep -m 1 "^${2} = ")" echo "${_ret#${2} = }" } @@ -499,7 +499,7 @@ arch_repo_add() { pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" - /usr/bin/repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ + repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ || error "repo-add -f ${repo}${FILESEXT} ${pkgs[@]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" @@ -518,7 +518,7 @@ arch_repo_remove() { fi repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ || error "repo-remove ${dbfile} ${pkgs[@]}" - /usr/bin/repo-remove -q "${filesfile}" ${pkgs[@]} \ + repo-remove -q "${filesfile}" ${pkgs[@]} \ || error "repo-remove ${filesfile} ${pkgs[@]}" set_repo_permission "${repo}" "${arch}" } diff --git a/db-update b/db-update index 6bd0209..7604547 100755 --- a/db-update +++ b/db-update @@ -39,9 +39,10 @@ for repo in ${repos[@]}; do # die "Package ${repo}/$(basename ${pkg}) already exists in another repository" #fi done - if ! check_splitpkgs ${repo} ${pkgs[@]}; then - die "Missing split packages for ${repo}" - fi + # This is fucking obnoxious +# if ! check_splitpkgs ${repo} ${pkgs[@]}; then +# die "Missing split packages for ${repo}" +# fi else die "Could not read ${STAGING}" fi diff --git a/get_license.sh b/get_license.sh deleted file mode 100755 index 1b755c6..0000000 --- a/get_license.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -# -*- coding: utf-8 -*- - - # get_license.sh - # Copyright 2010 Joshua Ismael Haase Hernández - - # ---------- GNU General Public License 3 ---------- - - # This file is part of Parabola. - - # Parabola is free software: you can redistribute it and/or modify - # it under the terms of the GNU General Public License as published by - # the Free Software Foundation, either version 3 of the License, or - # (at your option) any later version. - - # Parabola is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of - # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - # GNU General Public License for more details. - - # You should have received a copy of the GNU General Public License - # along with Parabola. If not, see . - -source ./config -source ./local_config -source ./libremessages - -msg "Creating pending licenses list" -pushd ${licenses_dir} >/dev/null -rm -rf ${licenses_dir}/* -popd >/dev/null - -dir=$(mktemp -d ${tempdir}/licenses.XXXX) -pushd $dir > /dev/null - -for repo in ${ARCHREPOS[@]}; do - msg2 "Extracting licenses in ${repo}" - pending=($(cut -d: -f1 ${docs_dir}/pending-${repo}.txt)) - for name in ${pending[@]}; do - plain "${pkg}" - for pkg in $(find ${repodir}/staging/${repo} -name "${name}-*${PKGEXT}" -printf '%f '); do - chmod +r ${pkg} - bsdtar -xf ${pkg} usr/share/licenses || { - error "${pkg} has no licenses" - } - chmod -r ${pkg} - done - done -done - -mv ${dir}/* ${licenses_dir}/ -rm -rf ${dir} - -exit 0 diff --git a/repo-update b/repo-update index 697e188..88ddda3 100755 --- a/repo-update +++ b/repo-update @@ -1,12 +1,12 @@ #!/bin/bash # -*- coding: utf-8 -*- -source config -source local_config -source libremessages +source $(dirname $0)/config +source $(dirname $0)/local_config +source $(dirname $0)/libremessages for repo in ${ARCHREPOS[@]}; do - for arch in 'i686' 'x86_64' 'any'; do + for arch in ${ARCHARCHES[@]} 'any'; do msg "Syncing ${repo} ${arch}" # makes a file containing rsync output for filter.py ${rsync_list_command} \ @@ -27,7 +27,7 @@ for repo in ${ARCHREPOS[@]}; do rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ ${repodir}/staging/${repo}/ done - for arch in 'i686' 'x86_64'; do + for arch in ${ARCHARCHES[@]}; do msg "Cleaning $repo $arch" # if there is a db in repo (db is created on rsync) if [ -r ${repodir}/staging/${repo}/os/${arch}/${repo}${DBEXT} ]; then @@ -43,7 +43,8 @@ for repo in ${ARCHREPOS[@]}; do python clean_repo.py -k ${blacklist} -d ${repodir}/staging/${repo} done -db-update -~/repm/cron-jobs/ftpdir-cleanup +msg "Removing leftover files..." +find ${repodir}/staging/ \! -name "*${PKGEXT}*" -delete -get_license.sh +db-update +$(dirname $0)/cron-jobs/ftpdir-cleanup -- cgit v1.2.3-2-g168b From 65cb50c483621d6043685513531001a24228062b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 22 Apr 2011 01:04:55 -0700 Subject: Clean_repo deletes pending-files --- clean_repo.py | 10 ++++++---- repo-update | 10 ++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/clean_repo.py b/clean_repo.py index 51e62d4..cc8e811 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -88,10 +88,12 @@ if __name__ == "__main__": if args.database: whitelisted=listado(args.whitelist_file) pkgs=pkginfo_from_db(args.database) - # remove_from_blacklist(args.database, blacklisted) - mkpending(pkgs, args.pending_file, - blacklisted, whitelisted) + pending_names=[pkg["name"] for pkg in + mkpending(pkgs, args.pending_file, + blacklisted, whitelisted)] - if args.directory: + if args.directory and args.database: + cleanup_nonfree_in_dir(args.directory, (blacklisted + pending_names)) + elif args.directory: cleanup_nonfree_in_dir(args.directory, blacklisted) diff --git a/repo-update b/repo-update index 88ddda3..8ad5aef 100755 --- a/repo-update +++ b/repo-update @@ -6,8 +6,9 @@ source $(dirname $0)/local_config source $(dirname $0)/libremessages for repo in ${ARCHREPOS[@]}; do + msg "Syncing ${repo}" for arch in ${ARCHARCHES[@]} 'any'; do - msg "Syncing ${repo} ${arch}" + msg2 "${repo} ${arch}" # makes a file containing rsync output for filter.py ${rsync_list_command} \ rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ @@ -28,18 +29,19 @@ for repo in ${ARCHREPOS[@]}; do ${repodir}/staging/${repo}/ done for arch in ${ARCHARCHES[@]}; do - msg "Cleaning $repo $arch" + msg2 "Making pending list for $repo $arch" # if there is a db in repo (db is created on rsync) if [ -r ${repodir}/staging/${repo}/os/${arch}/${repo}${DBEXT} ]; then # clean_repo makes pending list with files on db and remove # packages from db clean_repo.py -k ${blacklist} -w ${whitelist} \ -p ${docs_dir}/pending-${repo}.txt \ - -b ${repodir}/staging/${repo}/${repo}${DBEXT} + -b ${repodir}/staging/${repo}/${repo}${DBEXT} \ + -d ${repodir}/stagging/${repo} fi done # if some nonfree files got pass the filter this command delete them - msg "Fallback cleaning repo" + msg2 "Fallback cleaning repo" python clean_repo.py -k ${blacklist} -d ${repodir}/staging/${repo} done -- cgit v1.2.3-2-g168b From 63bf094a4e472b2bc15ed03fb956401b9246660e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 22 Apr 2011 18:22:15 -0700 Subject: repo-update source ~/.bashrc --- repo-update | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repo-update b/repo-update index 8ad5aef..d06b927 100755 --- a/repo-update +++ b/repo-update @@ -1,6 +1,6 @@ #!/bin/bash # -*- coding: utf-8 -*- - +source ~/.bashrc source $(dirname $0)/config source $(dirname $0)/local_config source $(dirname $0)/libremessages @@ -46,7 +46,7 @@ for repo in ${ARCHREPOS[@]}; do done msg "Removing leftover files..." -find ${repodir}/staging/ \! -name "*${PKGEXT}*" -delete +find ${repodir}/staging/ -type f \! -name "*${PKGEXT}" -delete db-update $(dirname $0)/cron-jobs/ftpdir-cleanup -- cgit v1.2.3-2-g168b From 4b2bd9a51f8f59106234f8889ae0ad98598f33bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 25 Apr 2011 15:54:01 -0700 Subject: * Added a cleaning cmd for symbolic links in staging * Added repo-add, repo-remove --- repo-add | 558 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ repo-remove | 505 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ repo-update | 10 +- 3 files changed, 1069 insertions(+), 4 deletions(-) create mode 100755 repo-add create mode 100755 repo-remove diff --git a/repo-add b/repo-add new file mode 100755 index 0000000..531d347 --- /dev/null +++ b/repo-add @@ -0,0 +1,558 @@ +#!/bin/bash +# +# repo-add - add a package to a given repo database file +# repo-remove - remove a package entry from a given repo database file +# Generated from repo-add.in; do not edit by hand. +# +# Copyright (c) 2006-2008 Aaron Griffin +# Copyright (c) 2007-2008 Dan McGee +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +LICENSESDIR=/home/parabolavnx/licenses + +# gettext initialization +export TEXTDOMAIN='pacman' +export TEXTDOMAINDIR='/usr/share/locale' + +myver='3.5.0' +confdir='/home/parabolavnx/etc' + +QUIET=0 +DELTA=0 +WITHFILES=0 +REPO_DB_FILE= +LOCKFILE= +CLEAN_LOCK=0 + +# ensure we have a sane umask set +umask 0022 + +msg() { + (( QUIET )) && return + local mesg=$1; shift + printf "==> ${mesg}\n" "$@" >&1 +} + +msg2() { + (( QUIET )) && return + local mesg=$1; shift + printf " -> ${mesg}\n" "$@" >&1 +} + +warning() { + local mesg=$1; shift + printf "==> $(gettext "WARNING:") ${mesg}\n" "$@" >&2 +} + +error() { + local mesg=$1; shift + printf "==> $(gettext "ERROR:") ${mesg}\n" "$@" >&2 +} + +# print usage instructions +usage() { + printf "repo-add, repo-remove (pacman) %s\n\n" "$myver" + printf "$(gettext "Usage: repo-add [-d] [-f] [-q] ...\n")" + printf "$(gettext "Usage: repo-remove [-q] ...\n\n")" + printf "$(gettext "\ +repo-add will update a package database by reading a package file.\n\ +Multiple packages to add can be specified on the command line.\n\n")" + printf "$(gettext "\ +repo-remove will update a package database by removing the package name\n\ +specified on the command line from the given repo database. Multiple\n\ +packages to remove can be specified on the command line.\n\n")" + printf "$(gettext "\ +Use the -q/--quiet flag to minimize output to basic messages, warnings,\n\ +and errors.\n\n")" + printf "$(gettext "\ +Use the -d/--delta flag to automatically generate and add a delta file\n\ +between the old entry and the new one, if the old package file is found\n\ +next to the new one.\n\n")" + printf "$(gettext "\ +Use the -f/--files flag to update a database including file entries.\n\n")" + echo "$(gettext "Example: repo-add /path/to/repo.db.tar.gz pacman-3.0.0.pkg.tar.gz")" + echo "$(gettext "Example: repo-remove /path/to/repo.db.tar.gz kernel26")" +} + +version() { + printf "repo-add, repo-remove (pacman) %s\n\n" "$myver" + printf "$(gettext "\ +Copyright (C) 2006-2008 Aaron Griffin .\n\ +Copyright (c) 2007-2008 Dan McGee .\n\n\ +This is free software; see the source for copying conditions.\n\ +There is NO WARRANTY, to the extent permitted by law.\n")" +} + +# write a list entry +# arg1 - Entry name +# arg2 - List +# arg3 - File to write to +write_list_entry() { + if [[ -n $2 ]]; then + echo "%$1%" >>$3 + echo -e $2 >>$3 + fi +} + +find_pkgentry() +{ + local pkgname=$1 + local pkgentry + for pkgentry in $tmpdir/$pkgname*; do + name=${pkgentry##*/} + if [[ ${name%-*-*} = $pkgname ]]; then + echo $pkgentry + return 0 + fi + done + return 1 +} + +# Get the package name from the delta filename +get_delta_pkgname() { + local tmp + + tmp=${1##*/} + echo ${tmp%-*-*_to*} +} + +# write a delta entry +# arg1 - path to delta file +db_write_delta() +{ + deltafile="$1" + pkgname="$(get_delta_pkgname $deltafile)" + + pkgentry=$(find_pkgentry $pkgname) + if [[ -z $pkgentry ]]; then + error "$(gettext "No database entry for package '%s'.")" "$pkgname" + return 1 + fi + deltas="$pkgentry/deltas" + if [[ ! -f $deltas ]]; then + echo -e "%DELTAS%" >$deltas + fi + # get md5sum and compressed size of package + md5sum="$(openssl dgst -md5 "$deltafile")" + md5sum="${md5sum##* }" + csize=$(stat -L -c %s "$deltafile") + + oldfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (source)" | sed 's/.*: *//') + newfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (output)" | sed 's/.*: *//') + + if grep -q "$oldfile.*$newfile" $deltas; then + sed -i.backup "/$oldfile.*$newfile/d" $deltas && rm -f $deltas.backup + fi + msg2 "$(gettext "Adding 'deltas' entry : %s -> %s")" "$oldfile" "$newfile" + echo ${deltafile##*/} $md5sum $csize $oldfile $newfile >> $deltas + + return 0 +} # end db_write_delta + +# remove a delta entry +# arg1 - path to delta file +db_remove_delta() +{ + deltafile="$1" + filename=${deltafile##*/} + pkgname="$(get_delta_pkgname $deltafile)" + + pkgentry=$(find_pkgentry $pkgname) + if [[ -z $pkgentry ]]; then + return 1 + fi + deltas="$pkgentry/deltas" + if [[ ! -f $deltas ]]; then + return 1 + fi + if grep -q "$filename" $deltas; then + sed -i.backup "/$filename/d" $deltas && rm -f $deltas.backup + msg2 "$(gettext "Removing existing entry '%s'...")" "$filename" + return 0 + fi + + return 1 +} # end db_remove_delta + +# write an entry to the pacman database +# arg1 - path to package +db_write_entry() +{ + # blank out all variables + local pkgfile="$1" + local pkgname pkgver pkgdesc csize size md5sum url arch builddate packager \ + _groups _licenses _replaces _depends _conflicts _provides _optdepends + + local OLDIFS="$IFS" + # IFS (field separator) is only the newline character + IFS=" +" + + # read info from the zipped package + local line var val + for line in $(bsdtar -xOqf "$pkgfile" .PKGINFO | + grep -v '^#' | sed 's|\(\w*\)\s*=\s*\(.*\)|\1 \2|'); do + # bash awesomeness here- var is always one word, val is everything else + var=${line%% *} + val=${line#* } + declare $var="$val" + case "$var" in + group) _groups="$_groups$group\n" ;; + license) _licenses="$_licenses$license\n" ;; + replaces) _replaces="$_replaces$replaces\n" ;; + depend) _depends="$_depends$depend\n" ;; + conflict) _conflicts="$_conflicts$conflict\n" ;; + provides) _provides="$_provides$provides\n" ;; + optdepend) _optdepends="$_optdepends$optdepend\n" ;; + esac + done + + IFS=$OLDIFS + + # get md5sum and compressed size of package + md5sum="$(openssl dgst -md5 "$pkgfile")" + md5sum="${md5sum##* }" + csize=$(stat -L -c %s "$pkgfile") + + # ensure $pkgname and $pkgver variables were found + if [[ -z $pkgname || -z $pkgver ]]; then + error "$(gettext "Invalid package file '%s'.")" "$pkgfile" + return 1 + fi + + pushd "$tmpdir" >/dev/null + if [[ -d $pkgname-$pkgver ]]; then + warning "$(gettext "An entry for '%s' already existed")" "$pkgname-$pkgver" + else + if (( DELTA )); then + pkgentry=$(find_pkgentry $pkgname) + if [[ -n $pkgentry ]]; then + local oldfilename=$(grep -A1 FILENAME $pkgentry/desc | tail -n1) + local oldfile="$(dirname $1)/$oldfilename" + fi + fi + fi + + # remove an existing entry if it exists, ignore failures + db_remove_entry "$pkgname" + + # create package directory + mkdir "$pkgname-$pkgver" + pushd "$pkgname-$pkgver" >/dev/null + + # restore an eventual deltas file + [[ -f ../$pkgname.deltas ]] && mv "../$pkgname.deltas" deltas + + # create desc entry + msg2 "$(gettext "Creating '%s' db entry...")" 'desc' + echo -e "%FILENAME%\n$(basename "$1")\n" >>desc + echo -e "%NAME%\n$pkgname\n" >>desc + [[ -n $pkgbase ]] && echo -e "%BASE%\n$pkgbase\n" >>desc + echo -e "%VERSION%\n$pkgver\n" >>desc + [[ -n $pkgdesc ]] && echo -e "%DESC%\n$pkgdesc\n" >>desc + write_list_entry "GROUPS" "$_groups" "desc" + [[ -n $csize ]] && echo -e "%CSIZE%\n$csize\n" >>desc + [[ -n $size ]] && echo -e "%ISIZE%\n$size\n" >>desc + + # compute checksums + msg2 "$(gettext "Computing md5 checksums...")" + echo -e "%MD5SUM%\n$md5sum\n" >>desc + + [[ -n $url ]] && echo -e "%URL%\n$url\n" >>desc + write_list_entry "LICENSE" "$_licenses" "desc" + [[ -n $arch ]] && echo -e "%ARCH%\n$arch\n" >>desc + [[ -n $builddate ]] && echo -e "%BUILDDATE%\n$builddate\n" >>desc + [[ -n $packager ]] && echo -e "%PACKAGER%\n$packager\n" >>desc + write_list_entry "REPLACES" "$_replaces" "desc" + + # create depends entry + msg2 "$(gettext "Creating '%s' db entry...")" 'depends' + # create the file even if it will remain empty + touch "depends" + write_list_entry "DEPENDS" "$_depends" "depends" + write_list_entry "CONFLICTS" "$_conflicts" "depends" + write_list_entry "PROVIDES" "$_provides" "depends" + write_list_entry "OPTDEPENDS" "$_optdepends" "depends" + + popd >/dev/null + popd >/dev/null + + # create files file if wanted + if (( WITHFILES )); then + msg2 "$(gettext "Creating '%s' db entry...")" 'files' + local files_path="$tmpdir/$pkgname-$pkgver/files" + echo "%FILES%" >$files_path + bsdtar --exclude='.*' -tf "$pkgfile" >>$files_path + fi + + # create a delta file + if (( DELTA )); then + if [[ -n $oldfilename ]]; then + if [[ -f $oldfile ]]; then + delta=$(pkgdelta -q $oldfile $1) + if [[ -f $delta ]]; then + db_write_delta $delta + fi + else + warning "$(gettext "Old package file not found: %s")" "$oldfilename" + fi + fi + fi + + # Extracts licenses to a common license dir + msg "Extracting license" + if [ -d ${LICENSESDIR}/${pkgname} ]; then + rm -r ${LICENSESDIR}/${pkgname} + fi + + # Change dir to licenses, and extract them stripping the first part of the path + bsdtar -C ${LICENSESDIR}/ --include="usr/share/licenses/" --strip-components 3 -xf ${pkgfile} >/dev/null 2>&1 + + if [ $? -ne 0 ]; then + warning "This package doesn't contain a license dir" + fi + + return 0 +} # end db_write_entry + +# remove existing entries from the DB +# arg1 - package name +db_remove_entry() { + local pkgname=$1 + local notfound=1 + local pkgentry=$(find_pkgentry $pkgname) + while [[ -n $pkgentry ]]; do + notfound=0 + if [[ -f $pkgentry/deltas ]]; then + mv "$pkgentry/deltas" "$tmpdir/$pkgname.deltas" + fi + msg2 "$(gettext "Removing existing entry '%s'...")" \ + "$(basename $pkgentry)" + rm -rf $pkgentry + pkgentry=$(find_pkgentry $pkgname) + done + + msg "Removing license" + if [ -d ${LICENSESDIR}/${pkgname} ]; then + rm -r ${LICENSESDIR}/${pkgname} + fi + + return $notfound +} # end db_remove_entry + +check_repo_db() +{ + # check lock file + if ( set -o noclobber; echo "$$" > "$LOCKFILE") 2> /dev/null; then + CLEAN_LOCK=1 + else + error "$(gettext "Failed to acquire lockfile: %s.")" "$LOCKFILE" + [[ -f $LOCKFILE ]] && error "$(gettext "Held by process %s")" "$(cat $LOCKFILE)" + exit 1 + fi + + if [[ -f $REPO_DB_FILE ]]; then + # there are two situations we can have here- a DB with some entries, + # or a DB with no contents at all. + if ! bsdtar -tqf "$REPO_DB_FILE" '*/desc' >/dev/null 2>&1; then + # check empty case + if [[ -n $(bsdtar -tqf "$REPO_DB_FILE" '*' 2>/dev/null) ]]; then + error "$(gettext "Repository file '%s' is not a proper pacman database.")" "$REPO_DB_FILE" + exit 1 + fi + fi + msg "$(gettext "Extracting database to a temporary location...")" + bsdtar -xf "$REPO_DB_FILE" -C "$tmpdir" + else + case "$cmd" in + repo-remove) + error "$(gettext "Repository file '%s' was not found.")" "$REPO_DB_FILE" + exit 1 + ;; + repo-add) + # check if the file can be created (write permission, directory existence, etc) + if ! touch "$REPO_DB_FILE"; then + error "$(gettext "Repository file '%s' could not be created.")" "$REPO_DB_FILE" + exit 1 + fi + rm -f "$REPO_DB_FILE" + ;; + esac + fi +} + +add() +{ + if [[ ! -f $1 ]]; then + error "$(gettext "File '%s' not found.")" "$1" + return 1 + fi + + if [[ ${1##*.} == "delta" ]]; then + deltafile=$1 + msg "$(gettext "Adding delta '%s'")" "$deltafile" + if ! type xdelta3 &>/dev/null; then + error "$(gettext "Cannot find the xdelta3 binary! Is xdelta3 installed?")" + exit 1 + fi + if db_write_delta "$deltafile"; then + return 0 + else + return 1 + fi + fi + + pkgfile=$1 + if ! bsdtar -tqf "$pkgfile" .PKGINFO >/dev/null 2>&1; then + error "$(gettext "'%s' is not a package file, skipping")" "$pkgfile" + return 1 + fi + + msg "$(gettext "Adding package '%s'")" "$pkgfile" + + db_write_entry "$pkgfile" +} + +remove() +{ + if [[ ${1##*.} == "delta" ]]; then + deltafile=$1 + msg "$(gettext "Searching for delta '%s'...")" "$deltafile" + if db_remove_delta "$deltafile"; then + return 0 + else + error "$(gettext "Delta matching '%s' not found.")" "$deltafile" + return 1 + fi + fi + + pkgname=$1 + msg "$(gettext "Searching for package '%s'...")" "$pkgname" + + if db_remove_entry "$pkgname"; then + rm -f "$tmpdir/$pkgname.deltas" + return 0 + else + error "$(gettext "Package matching '%s' not found.")" "$pkgname" + return 1 + fi +} + +trap_exit() +{ + echo + error "$@" + exit 1 +} + +clean_up() { + local exit_code=$? + + [[ -d $tmpdir ]] && rm -rf "$tmpdir" + (( CLEAN_LOCK )) && [[ -f $LOCKFILE ]] && rm -f "$LOCKFILE" + + exit $exit_code +} + +# PROGRAM START + +# determine whether we have gettext; make it a no-op if we do not +if ! type gettext &>/dev/null; then + gettext() { + echo "$@" + } +fi + +case "$1" in + -h|--help) usage; exit 0;; + -V|--version) version; exit 0;; +esac + +# figure out what program we are +cmd="$(basename $0)" +if [[ $cmd != "repo-add" && $cmd != "repo-remove" ]]; then + error "$(gettext "Invalid command name '%s' specified.")" "$cmd" + exit 1 +fi + +tmpdir=$(mktemp -d /tmp/repo-tools.XXXXXXXXXX) || (\ + error "$(gettext "Cannot create temp directory for database building.")"; \ + exit 1) + +trap 'clean_up' EXIT +trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT +trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT +trap 'trap_exit "$(gettext "An unknown error has occured. Exiting...")"' ERR + +success=0 +# parse arguments +for arg in "$@"; do + case "$arg" in + -q|--quiet) QUIET=1;; + -d|--delta) DELTA=1;; + -f|--files) WITHFILES=1;; + *) + if [[ -z $REPO_DB_FILE ]]; then + REPO_DB_FILE="$arg" + LOCKFILE="$REPO_DB_FILE.lck" + check_repo_db + else + case "$cmd" in + repo-add) add $arg && success=1 ;; + repo-remove) remove $arg && success=1 ;; + esac + fi + ;; + esac +done + +# if at least one operation was a success, re-zip database +if (( success )); then + msg "$(gettext "Creating updated database file '%s'")" "$REPO_DB_FILE" + + case "$REPO_DB_FILE" in + *tar.gz) TAR_OPT="z" ;; + *tar.bz2) TAR_OPT="j" ;; + *tar.xz) TAR_OPT="J" ;; + *) warning "$(gettext "'%s' does not have a valid archive extension.")" \ + "$REPO_DB_FILE" ;; + esac + + filename=$(basename "$REPO_DB_FILE") + + pushd "$tmpdir" >/dev/null + if [[ -n $(ls) ]]; then + bsdtar -c${TAR_OPT}f "$filename" * + else + # we have no packages remaining? zip up some emptyness + warning "$(gettext "No packages remain, creating empty database.")" + bsdtar -c${TAR_OPT}f "$filename" -T /dev/null + fi + popd >/dev/null + + [[ -f $REPO_DB_FILE ]] && mv -f "$REPO_DB_FILE" "${REPO_DB_FILE}.old" + [[ -f $tmpdir/$filename ]] && mv "$tmpdir/$filename" "$REPO_DB_FILE" + dblink="${REPO_DB_FILE%.tar.*}" + target=${REPO_DB_FILE##*/} + ln -sf "$target" "$dblink" 2>/dev/null || \ + ln -f "$target" "$dblink" 2>/dev/null || \ + cp "$REPO_DB_FILE" "$dblink" +else + msg "$(gettext "No packages modified, nothing to do.")" + exit 1 +fi + +exit 0 +# vim: set ts=2 sw=2 noet: diff --git a/repo-remove b/repo-remove new file mode 100755 index 0000000..ef6f186 --- /dev/null +++ b/repo-remove @@ -0,0 +1,505 @@ +#!/bin/bash +# +# repo-add - add a package to a given repo database file +# repo-remove - remove a package entry from a given repo database file +# Generated from repo-add.in; do not edit by hand. +# +# Copyright (c) 2006-2008 Aaron Griffin +# Copyright (c) 2007-2008 Dan McGee +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# gettext initialization +export TEXTDOMAIN='pacman' +export TEXTDOMAINDIR='/usr/share/locale' + +myver='3.3.3' +confdir='/etc' + +QUIET=0 +REPO_DB_FILE= +LOCKFILE= +CLEAN_LOCK=0 +startdir="$PWD" + +# ensure we have a sane umask set +umask 0022 + +msg() { + local mesg=$1; shift + printf "==> ${mesg}\n" "$@" >&1 +} + +msg2() { + [ $QUIET -ne 0 ] && return + local mesg=$1; shift + printf " -> ${mesg}\n" "$@" >&1 +} + +warning() { + local mesg=$1; shift + printf "==> $(gettext "WARNING:") ${mesg}\n" "$@" >&2 +} + +error() { + local mesg=$1; shift + printf "==> $(gettext "ERROR:") ${mesg}\n" "$@" >&2 +} + +# print usage instructions +usage() { + printf "repo-add, repo-remove (pacman) %s\n\n" "$myver" + printf "$(gettext "Usage: repo-add [-q] ...\n")" + printf "$(gettext "Usage: repo-remove [-q] ...\n\n")" + printf "$(gettext "\ +repo-add will update a package database by reading a package file.\n\ +Multiple packages to add can be specified on the command line.\n\n")" + printf "$(gettext "\ +repo-remove will update a package database by removing the package name\n\ +specified on the command line from the given repo database. Multiple\n\ +packages to remove can be specified on the command line.\n\n")" + printf "$(gettext "\ +Use the -q/--quiet flag to minimize output to basic messages, warnings,\n\ +and errors\n\n")" + echo "$(gettext "Example: repo-add /path/to/repo.db.tar.gz pacman-3.0.0.pkg.tar.gz")" + echo "$(gettext "Example: repo-remove /path/to/repo.db.tar.gz kernel26")" +} + +version() { + printf "repo-add, repo-remove (pacman) %s\n\n" "$myver" + printf "$(gettext "\ +Copyright (C) 2006-2008 Aaron Griffin .\n\ +Copyright (c) 2007-2008 Dan McGee .\n\n\ +This is free software; see the source for copying conditions.\n\ +There is NO WARRANTY, to the extent permitted by law.\n")" +} + +# write a list entry +# arg1 - Entry name +# arg2 - List +# arg3 - File to write to +write_list_entry() { + if [ -n "$2" ]; then + echo "%$1%" >>$3 + echo -e $2 >>$3 + fi +} + +find_pkgentry() +{ + local pkgname=$1 + local pkgentry + for pkgentry in $tmpdir/$pkgname*; do + name=${pkgentry##*/} + if [ "${name%-*-*}" = "$pkgname" ]; then + echo $pkgentry + return 0 + fi + done + return 1 +} + +# Get the package name from the delta filename +get_delta_pkgname() { + local tmp + + tmp=${1##*/} + echo ${tmp%-*-*_to*} +} + +# write a delta entry +# arg1 - path to delta file +db_write_delta() +{ + deltafile="$1" + pkgname="$(get_delta_pkgname $deltafile)" + + pkgentry=$(find_pkgentry $pkgname) + if [ -z "$pkgentry" ]; then + return 1 + fi + deltas="$pkgentry/deltas" + # create deltas file if it does not already exist + if [ ! -f "$deltas" ]; then + msg2 "$(gettext "Creating 'deltas' db entry...")" + echo -e "%DELTAS%" >>$deltas + fi + # get md5sum and compressed size of package + md5sum="$(openssl dgst -md5 "$deltafile" | awk '{print $NF}')" + csize=$(stat -L -c %s "$deltafile") + + oldfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (source)" | sed 's/.*: *//') + newfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (output)" | sed 's/.*: *//') + + if grep -q "$oldfile.*$newfile" $deltas; then + warning "$(gettext "An entry for '%s' already existed")" "$deltafile" + sed -i.backup "/$oldfile.*$newfile/d" $deltas && rm -f $deltas.backup + msg2 "$(gettext "Removing existing entry '%s'...")" "$deltafile" + fi + echo ${deltafile##*/} $md5sum $csize $oldfile $newfile >> $deltas + + return 0 +} # end db_write_delta + +# remove a delta entry +# arg1 - path to delta file +db_remove_delta() +{ + deltafile="$1" + filename=${deltafile##*/} + pkgname="$(get_delta_pkgname $deltafile)" + + pkgentry=$(find_pkgentry $pkgname) + if [ -z "$pkgentry" ]; then + return 1 + fi + deltas="$pkgentry/deltas" + if [ ! -f "$deltas" ]; then + return 1 + fi + if grep -q "$filename" $deltas; then + sed -i.backup "/$filename/d" $deltas && rm -f $deltas.backup + msg2 "$(gettext "Removing existing entry '%s'...")" "$filename" + return 0 + fi + + return 1 +} # end db_remove_delta + +# write an entry to the pacman database +# arg1 - path to package +db_write_entry() +{ + # blank out all variables + local pkgfile="$1" + local pkgname pkgver pkgdesc csize size md5sum url arch builddate packager force \ + _groups _licenses _replaces _depends _conflicts _provides _optdepends + + local OLDIFS="$IFS" + # IFS (field separator) is only the newline character + IFS=" +" + + # read info from the zipped package + local line var val + for line in $(bsdtar -xOqf "$pkgfile" .PKGINFO | + grep -v '^#' | sed 's|\(\w*\)\s*=\s*\(.*\)|\1 \2|'); do + # bash awesomeness here- var is always one word, val is everything else + var=${line%% *} + val=${line#* } + declare $var="$val" + case "$var" in + group) _groups="$_groups$group\n" ;; + license) _licenses="$_licenses$license\n" ;; + replaces) _replaces="$_replaces$replaces\n" ;; + depend) _depends="$_depends$depend\n" ;; + conflict) _conflicts="$_conflicts$conflict\n" ;; + provides) _provides="$_provides$provides\n" ;; + optdepend) _optdepends="$_optdepends$optdepend\n" ;; + esac + done + + IFS=$OLDIFS + + # get md5sum and compressed size of package + md5sum="$(openssl dgst -md5 "$pkgfile" | awk '{print $NF}')" + csize=$(stat -L -c %s "$pkgfile") + + # ensure $pkgname and $pkgver variables were found + if [ -z "$pkgname" -o -z "$pkgver" ]; then + error "$(gettext "Invalid package file '%s'.")" "$pkgfile" + return 1 + fi + + cd "$tmpdir" + + if [ -d "$pkgname-$pkgver" ]; then + warning "$(gettext "An entry for '%s' already existed")" "$pkgname-$pkgver" + fi + + # remove an existing entry if it exists, ignore failures + db_remove_entry "$pkgname" + + # create package directory + mkdir "$pkgname-$pkgver" + cd "$pkgname-$pkgver" + + # restore an eventual deltas file + [ -f "../$pkgname.deltas" ] && mv "../$pkgname.deltas" deltas + + # create desc entry + msg2 "$(gettext "Creating 'desc' db entry...")" + echo -e "%FILENAME%\n$(basename "$1")\n" >>desc + echo -e "%NAME%\n$pkgname\n" >>desc + [ -n "$pkgbase" ] && echo -e "%BASE%\n$pkgbase\n" >>desc + echo -e "%VERSION%\n$pkgver\n" >>desc + [ -n "$pkgdesc" ] && echo -e "%DESC%\n$pkgdesc\n" >>desc + write_list_entry "GROUPS" "$_groups" "desc" + [ -n "$csize" ] && echo -e "%CSIZE%\n$csize\n" >>desc + [ -n "$size" ] && echo -e "%ISIZE%\n$size\n" >>desc + + # compute checksums + msg2 "$(gettext "Computing md5 checksums...")" + echo -e "%MD5SUM%\n$md5sum\n" >>desc + + [ -n "$url" ] && echo -e "%URL%\n$url\n" >>desc + write_list_entry "LICENSE" "$_licenses" "desc" + [ -n "$arch" ] && echo -e "%ARCH%\n$arch\n" >>desc + [ -n "$builddate" ] && echo -e "%BUILDDATE%\n$builddate\n" >>desc + [ -n "$packager" ] && echo -e "%PACKAGER%\n$packager\n" >>desc + write_list_entry "REPLACES" "$_replaces" "desc" + [ -n "$force" ] && echo -e "%FORCE%\n" >>desc + + # create depends entry + msg2 "$(gettext "Creating 'depends' db entry...")" + # create the file even if it will remain empty + touch "depends" + write_list_entry "DEPENDS" "$_depends" "depends" + write_list_entry "CONFLICTS" "$_conflicts" "depends" + write_list_entry "PROVIDES" "$_provides" "depends" + write_list_entry "OPTDEPENDS" "$_optdepends" "depends" + + cd "$startdir" + + return 0 +} # end db_write_entry + +# remove existing entries from the DB +# arg1 - package name +db_remove_entry() { + local pkgname=$1 + local notfound=1 + local pkgentry=$(find_pkgentry $pkgname) + while [ -n "$pkgentry" ]; do + notfound=0 + if [ -f "$pkgentry/deltas" ]; then + mv "$pkgentry/deltas" "$tmpdir/$pkgname.deltas" + fi + msg2 "$(gettext "Removing existing entry '%s'...")" \ + "$(basename $pkgentry)" + rm -rf $pkgentry + pkgentry=$(find_pkgentry $pkgname) + done + return $notfound +} # end db_remove_entry + +check_repo_db() +{ + # check lock file + if ( set -o noclobber; echo "$$" > "$LOCKFILE") 2> /dev/null; then + CLEAN_LOCK=1 + else + error "$(gettext "Failed to acquire lockfile: %s.")" "$LOCKFILE" + [ -f "$LOCKFILE" ] && error "$(gettext "Held by process %s")" "$(cat $LOCKFILE)" + exit 1 + fi + + if [ -f "$REPO_DB_FILE" ]; then + # there are two situations we can have here- a DB with some entries, + # or a DB with no contents at all. + if ! bsdtar -tqf "$REPO_DB_FILE" '*/desc' >/dev/null 2>&1; then + # check empty case + if [ -n "$(bsdtar -tqf "$REPO_DB_FILE" '*' 2>/dev/null)" ]; then + error "$(gettext "Repository file '%s' is not a proper pacman database.")" "$REPO_DB_FILE" + exit 1 + fi + fi + msg "$(gettext "Extracting database to a temporary location...")" + bsdtar -xf "$REPO_DB_FILE" -C "$tmpdir" + else + case "$cmd" in + repo-remove) + error "$(gettext "Repository file '%s' was not found.")" "$REPO_DB_FILE" + exit 1 + ;; + repo-add) + # check if the file can be created (write permission, directory existence, etc) + if ! touch "$REPO_DB_FILE"; then + error "$(gettext "Repository file '%s' could not be created.")" "$REPO_DB_FILE" + exit 1 + fi + rm -f "$REPO_DB_FILE" + ;; + esac + fi +} + +add() +{ + if [ ! -f "$1" ]; then + error "$(gettext "File '%s' not found.")" "$1" + return 1 + fi + + if [ "${1##*.}" == "delta" ]; then + deltafile=$1 + msg "$(gettext "Adding delta '%s'")" "$deltafile" + if [ ! "$(type -p xdelta3)" ]; then + error "$(gettext "Cannot find the xdelta3 binary! Is xdelta3 installed?")" + exit 1 + fi + if db_write_delta "$deltafile"; then + return 0 + else + return 1 + fi + fi + + pkgfile=$1 + if ! bsdtar -tqf "$pkgfile" .PKGINFO 2>&1 >/dev/null; then + error "$(gettext "'%s' is not a package file, skipping")" "$pkgfile" + return 1 + fi + + msg "$(gettext "Adding package '%s'")" "$pkgfile" + + db_write_entry "$pkgfile" +} + +remove() +{ + if [ "${1##*.}" == "delta" ]; then + deltafile=$1 + msg "$(gettext "Searching for delta '%s'...")" "$deltafile" + if db_remove_delta "$deltafile"; then + return 0 + else + error "$(gettext "Delta matching '%s' not found.")" "$deltafile" + return 1 + fi + fi + + pkgname=$1 + msg "$(gettext "Searching for package '%s'...")" "$pkgname" + + if db_remove_entry "$pkgname"; then + rm -f "$tmpdir/$pkgname.deltas" + return 0 + else + error "$(gettext "Package matching '%s' not found.")" "$pkgname" + return 1 + fi +} + +trap_exit() +{ + echo + error "$@" + exit 1 +} + +clean_up() { + local exit_code=$? + + cd "$startdir" + [ -d "$tmpdir" ] && rm -rf "$tmpdir" + [ $CLEAN_LOCK -eq 1 -a -f "$LOCKFILE" ] && rm -f "$LOCKFILE" + + exit $exit_code +} + +# PROGRAM START + +# determine whether we have gettext; make it a no-op if we do not +if [ ! $(type -t gettext) ]; then + gettext() { + echo "$@" + } +fi + +case "$1" in + -h|--help) usage; exit 0;; + -V|--version) version; exit 0;; +esac + +# check for correct number of args +if [ $# -lt 2 ]; then + usage + exit 1 +fi + +# figure out what program we are +cmd="$(basename $0)" +if [ "$cmd" != "repo-add" -a "$cmd" != "repo-remove" ]; then + error "$(gettext "Invalid command name '%s' specified.")" "$cmd" + exit 1 +fi + +tmpdir=$(mktemp -d /tmp/repo-tools.XXXXXXXXXX) || (\ + error "$(gettext "Cannot create temp directory for database building.")"; \ + exit 1) + +trap 'clean_up' EXIT +trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT +trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT +trap 'trap_exit "$(gettext "An unknown error has occured. Exiting...")"' ERR + +success=0 +# parse arguments +for arg in "$@"; do + case "$arg" in + -q|--quiet) QUIET=1;; + + -f|--force) + warning "$(gettext "the -f and --force options are no longer recognized")" + msg2 "$(gettext "use options=(force) in the PKGBUILD instead")" + ;; + + *) + if [ -z "$REPO_DB_FILE" ]; then + REPO_DB_FILE="$arg" + LOCKFILE="$REPO_DB_FILE.lck" + check_repo_db + else + case "$cmd" in + repo-add) add $arg && success=1 ;; + repo-remove) remove $arg && success=1 ;; + esac + fi + ;; + esac +done + +# if at least one operation was a success, re-zip database +if [ $success -eq 1 ]; then + msg "$(gettext "Creating updated database file '%s'")" "$REPO_DB_FILE" + + case "$REPO_DB_FILE" in + *tar.gz) TAR_OPT="z" ;; + *tar.bz2) TAR_OPT="j" ;; + *tar.xz) TAR_OPT="J" ;; + *) warning "$(gettext "'%s' does not have a valid archive extension.")" \ + "$REPO_DB_FILE" ;; + esac + + filename=$(basename "$REPO_DB_FILE") + + cd "$tmpdir" + if [ -n "$(ls)" ]; then + bsdtar -c${TAR_OPT}f "$filename" * + else + # we have no packages remaining? zip up some emptyness + warning "$(gettext "No packages remain, creating empty database.")" + bsdtar -c${TAR_OPT}f "$filename" -T /dev/null + fi + cd "$startdir" + + [ -f "$REPO_DB_FILE" ] && mv -f "$REPO_DB_FILE" "${REPO_DB_FILE}.old" + [ -f "$tmpdir/$filename" ] && mv "$tmpdir/$filename" "$REPO_DB_FILE" +else + msg "$(gettext "No packages modified, nothing to do.")" +fi + +exit 0 +# vim: set ts=2 sw=2 noet: diff --git a/repo-update b/repo-update index d06b927..60960f2 100755 --- a/repo-update +++ b/repo-update @@ -34,7 +34,7 @@ for repo in ${ARCHREPOS[@]}; do if [ -r ${repodir}/staging/${repo}/os/${arch}/${repo}${DBEXT} ]; then # clean_repo makes pending list with files on db and remove # packages from db - clean_repo.py -k ${blacklist} -w ${whitelist} \ + $(dirname $0)/clean_repo.py -k ${blacklist} -w ${whitelist} \ -p ${docs_dir}/pending-${repo}.txt \ -b ${repodir}/staging/${repo}/${repo}${DBEXT} \ -d ${repodir}/stagging/${repo} @@ -42,11 +42,13 @@ for repo in ${ARCHREPOS[@]}; do done # if some nonfree files got pass the filter this command delete them msg2 "Fallback cleaning repo" - python clean_repo.py -k ${blacklist} -d ${repodir}/staging/${repo} + $(dirname $0)/clean_repo.py -k ${blacklist} -d ${repodir}/staging/${repo} done msg "Removing leftover files..." find ${repodir}/staging/ -type f \! -name "*${PKGEXT}" -delete +# Staging should not have symbolic links +find ${repodir}/staging/ -type l -delete -db-update -$(dirname $0)/cron-jobs/ftpdir-cleanup +$(dirname $0)/db-update +#$(dirname $0)/cron-jobs/ftpdir-cleanup -- cgit v1.2.3-2-g168b From 74093d95cba935badaf7572e1dcd38ffd76bdc53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 8 May 2011 18:45:02 -0500 Subject: * Remove non free packages on db-update --- config | 1 + db-update | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/config b/config index 6936a9b..effdde4 100644 --- a/config +++ b/config @@ -33,3 +33,4 @@ PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" MAKEPKGCONF="$HOME/etc/makepkg.conf" +BLACKLIST_FILE="$HOME/parabolagnulinux.org/docs/blacklist.txt" \ No newline at end of file diff --git a/db-update b/db-update index 7604547..6c220d9 100755 --- a/db-update +++ b/db-update @@ -69,7 +69,7 @@ for repo in ${repos[@]}; do if [ -f "$FTP_BASE/${PKGPOOL}/${pkgfile}.sig" ]; then ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "$FTP_BASE/$repo/os/${pkgarch}" fi - add_pkgs[${#add_pkgs[*]}]=${pkgfile} + add_pkgs[${#add_pkgs[*]}]=${pkgfile} done if [ ${#add_pkgs[@]} -ge 1 ]; then arch_repo_add "${repo}" "${pkgarch}" ${add_pkgs[@]} @@ -77,6 +77,21 @@ for repo in ${repos[@]}; do done done +# Repo check nonfree +nonfree=($(cut -d: -f1 ${BLACKLIST_FILE})) +for repo in ${ARCHREPOS[@]}; do + for pkgarch in ${ARCHARCHES[@]}; do + cleanpkgs=() + dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort )) + for pkgname in ${dbpkgs[@]}; do + if in_array ${pkgname} ${nonfree[@]}; then + cleanpkgs[${#cleanpkgs[*]}]=${pkgname} + fi + done + arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]} + done +done + for repo in ${repos[@]}; do for pkgarch in ${ARCHES[@]}; do repo_unlock ${repo} ${pkgarch} -- cgit v1.2.3-2-g168b From 3e7ae87a5a2ac70811a016d684a8a24ae18f49f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 8 May 2011 19:04:39 -0700 Subject: * Unique pkgname list for nonfree and dbpkgs --- db-update | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/db-update b/db-update index 6c220d9..f077aff 100755 --- a/db-update +++ b/db-update @@ -78,17 +78,24 @@ for repo in ${repos[@]}; do done # Repo check nonfree -nonfree=($(cut -d: -f1 ${BLACKLIST_FILE})) +nonfree=($(cut -d: -f1 ${BLACKLIST_FILE} | sort -u)) for repo in ${ARCHREPOS[@]}; do - for pkgarch in ${ARCHARCHES[@]}; do + for pkgarch in ${ARCHES[@]}; do + if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then + continue + fi + unset dbpkgs + unset cleanpkgs cleanpkgs=() - dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort )) + dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) for pkgname in ${dbpkgs[@]}; do if in_array ${pkgname} ${nonfree[@]}; then cleanpkgs[${#cleanpkgs[*]}]=${pkgname} fi done - arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]} + if [ ${#cleanpkgs[@]} -ge 1 ]; then + arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]} + fi done done @@ -97,3 +104,5 @@ for repo in ${repos[@]}; do repo_unlock ${repo} ${pkgarch} done done + + -- cgit v1.2.3-2-g168b From e6e9e1a468c15a6ca609d8c6072a7b8c16fb3833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 15 May 2011 22:01:02 -0500 Subject: db-update msg when cleaning nonfree --- db-update | 2 ++ 1 file changed, 2 insertions(+) diff --git a/db-update b/db-update index f077aff..1606f1d 100755 --- a/db-update +++ b/db-update @@ -78,9 +78,11 @@ for repo in ${repos[@]}; do done # Repo check nonfree +msg "Check nonfree in repo:" nonfree=($(cut -d: -f1 ${BLACKLIST_FILE} | sort -u)) for repo in ${ARCHREPOS[@]}; do for pkgarch in ${ARCHES[@]}; do + msg2 "$repo $pkgarch" if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then continue fi -- cgit v1.2.3-2-g168b From b88a0166be595545c2b2d68451291f8ba6bb08ab Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernandez Date: Wed, 18 May 2011 21:21:59 -0500 Subject: * Separated nonfree check from db-update * Added db-check-nonfree to repo-update --- db-check-nonfree | 45 +++++++++++++++++++++++++++++++++++++++++++++ db-update | 24 ------------------------ repo-update | 3 ++- 3 files changed, 47 insertions(+), 25 deletions(-) create mode 100644 db-check-nonfree diff --git a/db-check-nonfree b/db-check-nonfree new file mode 100644 index 0000000..83efb14 --- /dev/null +++ b/db-check-nonfree @@ -0,0 +1,45 @@ +#!/bin/bash + +. "$(dirname $0)/db-functions" +. "$(dirname $0)/config" + +if [ $# -ge 1 ]; then + warning "Calling $(basename $0) with a specific repository is not supported" + exit 1 +fi + +# TODO: this might lock too much (architectures) +for repo in ${repos[@]}; do + for pkgarch in ${ARCHES[@]}; do + repo_lock ${repo} ${pkgarch} || exit 1 + done +done + +msg "Check nonfree in repo:" +nonfree=($(cut -d: -f1 ${BLACKLIST_FILE} | sort -u)) +for repo in ${ARCHREPOS[@]}; do + for pkgarch in ${ARCHES[@]}; do + msg2 "$repo $pkgarch" + if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then + continue + fi + unset dbpkgs + unset cleanpkgs + cleanpkgs=() + dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) + for pkgname in ${dbpkgs[@]}; do + if in_array ${pkgname} ${nonfree[@]}; then + cleanpkgs[${#cleanpkgs[*]}]=${pkgname} + fi + done + if [ ${#cleanpkgs[@]} -ge 1 ]; then + arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]} + fi + done +done + +for repo in ${repos[@]}; do + for pkgarch in ${ARCHES[@]}; do + repo_unlock ${repo} ${pkgarch} + done +done diff --git a/db-update b/db-update index 1606f1d..86eaa2e 100755 --- a/db-update +++ b/db-update @@ -77,30 +77,6 @@ for repo in ${repos[@]}; do done done -# Repo check nonfree -msg "Check nonfree in repo:" -nonfree=($(cut -d: -f1 ${BLACKLIST_FILE} | sort -u)) -for repo in ${ARCHREPOS[@]}; do - for pkgarch in ${ARCHES[@]}; do - msg2 "$repo $pkgarch" - if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then - continue - fi - unset dbpkgs - unset cleanpkgs - cleanpkgs=() - dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) - for pkgname in ${dbpkgs[@]}; do - if in_array ${pkgname} ${nonfree[@]}; then - cleanpkgs[${#cleanpkgs[*]}]=${pkgname} - fi - done - if [ ${#cleanpkgs[@]} -ge 1 ]; then - arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]} - fi - done -done - for repo in ${repos[@]}; do for pkgarch in ${ARCHES[@]}; do repo_unlock ${repo} ${pkgarch} diff --git a/repo-update b/repo-update index 60960f2..a44ae87 100755 --- a/repo-update +++ b/repo-update @@ -51,4 +51,5 @@ find ${repodir}/staging/ -type f \! -name "*${PKGEXT}" -delete find ${repodir}/staging/ -type l -delete $(dirname $0)/db-update -#$(dirname $0)/cron-jobs/ftpdir-cleanup +$(dirname $0)/db-check-nonfree + -- cgit v1.2.3-2-g168b From 8766129748c8ad165b4245f53df1a2466be30848 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Wed, 18 May 2011 19:28:28 -0700 Subject: db-check-nonfree is executable --- db-check-nonfree | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 db-check-nonfree diff --git a/db-check-nonfree b/db-check-nonfree old mode 100644 new mode 100755 -- cgit v1.2.3-2-g168b From 303dba7c0ab15cd4e726fe3f72ccf8347df84d10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 10 Jun 2011 16:06:11 -0500 Subject: * repo-add only extract license if package has "custom" license. * repo-remove hardlink to repo-add. * repo-update has some more info output and do not display rsync errors. --- repo-add | 23 ++++--- repo-remove | 204 ++++++++++++++++++++++++++++++++++++++---------------------- repo-update | 8 ++- 3 files changed, 149 insertions(+), 86 deletions(-) diff --git a/repo-add b/repo-add index 531d347..c4bf96f 100755 --- a/repo-add +++ b/repo-add @@ -313,18 +313,21 @@ db_write_entry() # Extracts licenses to a common license dir msg "Extracting license" - if [ -d ${LICENSESDIR}/${pkgname} ]; then - rm -r ${LICENSESDIR}/${pkgname} - fi + if bsdtar -xOf ${pkgfile} .PKGINFO | grep "license" | grep "custom" ; then + if [ -d ${LICENSESDIR}/${pkgname} ]; then + rm -r ${LICENSESDIR}/${pkgname} + fi # Change dir to licenses, and extract them stripping the first part of the path - bsdtar -C ${LICENSESDIR}/ --include="usr/share/licenses/" --strip-components 3 -xf ${pkgfile} >/dev/null 2>&1 + bsdtar -C ${LICENSESDIR}/ --include="usr/share/licenses/" \ + --strip-components 3 -xf ${pkgfile} >/dev/null 2>&1 - if [ $? -ne 0 ]; then + if [ $? -ne 0 ]; then warning "This package doesn't contain a license dir" - fi - - return 0 + fi + fi + + return 0 } # end db_write_entry # remove existing entries from the DB @@ -346,8 +349,8 @@ db_remove_entry() { msg "Removing license" if [ -d ${LICENSESDIR}/${pkgname} ]; then - rm -r ${LICENSESDIR}/${pkgname} - fi + rm -r ${LICENSESDIR}/${pkgname} + fi return $notfound } # end db_remove_entry diff --git a/repo-remove b/repo-remove index ef6f186..c4bf96f 100755 --- a/repo-remove +++ b/repo-remove @@ -20,29 +20,33 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +LICENSESDIR=/home/parabolavnx/licenses + # gettext initialization export TEXTDOMAIN='pacman' export TEXTDOMAINDIR='/usr/share/locale' -myver='3.3.3' -confdir='/etc' +myver='3.5.0' +confdir='/home/parabolavnx/etc' QUIET=0 +DELTA=0 +WITHFILES=0 REPO_DB_FILE= LOCKFILE= CLEAN_LOCK=0 -startdir="$PWD" # ensure we have a sane umask set umask 0022 msg() { + (( QUIET )) && return local mesg=$1; shift printf "==> ${mesg}\n" "$@" >&1 } msg2() { - [ $QUIET -ne 0 ] && return + (( QUIET )) && return local mesg=$1; shift printf " -> ${mesg}\n" "$@" >&1 } @@ -60,7 +64,7 @@ error() { # print usage instructions usage() { printf "repo-add, repo-remove (pacman) %s\n\n" "$myver" - printf "$(gettext "Usage: repo-add [-q] ...\n")" + printf "$(gettext "Usage: repo-add [-d] [-f] [-q] ...\n")" printf "$(gettext "Usage: repo-remove [-q] ...\n\n")" printf "$(gettext "\ repo-add will update a package database by reading a package file.\n\ @@ -71,7 +75,13 @@ specified on the command line from the given repo database. Multiple\n\ packages to remove can be specified on the command line.\n\n")" printf "$(gettext "\ Use the -q/--quiet flag to minimize output to basic messages, warnings,\n\ -and errors\n\n")" +and errors.\n\n")" + printf "$(gettext "\ +Use the -d/--delta flag to automatically generate and add a delta file\n\ +between the old entry and the new one, if the old package file is found\n\ +next to the new one.\n\n")" + printf "$(gettext "\ +Use the -f/--files flag to update a database including file entries.\n\n")" echo "$(gettext "Example: repo-add /path/to/repo.db.tar.gz pacman-3.0.0.pkg.tar.gz")" echo "$(gettext "Example: repo-remove /path/to/repo.db.tar.gz kernel26")" } @@ -90,7 +100,7 @@ There is NO WARRANTY, to the extent permitted by law.\n")" # arg2 - List # arg3 - File to write to write_list_entry() { - if [ -n "$2" ]; then + if [[ -n $2 ]]; then echo "%$1%" >>$3 echo -e $2 >>$3 fi @@ -102,7 +112,7 @@ find_pkgentry() local pkgentry for pkgentry in $tmpdir/$pkgname*; do name=${pkgentry##*/} - if [ "${name%-*-*}" = "$pkgname" ]; then + if [[ ${name%-*-*} = $pkgname ]]; then echo $pkgentry return 0 fi @@ -126,27 +136,26 @@ db_write_delta() pkgname="$(get_delta_pkgname $deltafile)" pkgentry=$(find_pkgentry $pkgname) - if [ -z "$pkgentry" ]; then + if [[ -z $pkgentry ]]; then + error "$(gettext "No database entry for package '%s'.")" "$pkgname" return 1 fi deltas="$pkgentry/deltas" - # create deltas file if it does not already exist - if [ ! -f "$deltas" ]; then - msg2 "$(gettext "Creating 'deltas' db entry...")" - echo -e "%DELTAS%" >>$deltas + if [[ ! -f $deltas ]]; then + echo -e "%DELTAS%" >$deltas fi # get md5sum and compressed size of package - md5sum="$(openssl dgst -md5 "$deltafile" | awk '{print $NF}')" + md5sum="$(openssl dgst -md5 "$deltafile")" + md5sum="${md5sum##* }" csize=$(stat -L -c %s "$deltafile") oldfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (source)" | sed 's/.*: *//') newfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (output)" | sed 's/.*: *//') if grep -q "$oldfile.*$newfile" $deltas; then - warning "$(gettext "An entry for '%s' already existed")" "$deltafile" sed -i.backup "/$oldfile.*$newfile/d" $deltas && rm -f $deltas.backup - msg2 "$(gettext "Removing existing entry '%s'...")" "$deltafile" fi + msg2 "$(gettext "Adding 'deltas' entry : %s -> %s")" "$oldfile" "$newfile" echo ${deltafile##*/} $md5sum $csize $oldfile $newfile >> $deltas return 0 @@ -161,11 +170,11 @@ db_remove_delta() pkgname="$(get_delta_pkgname $deltafile)" pkgentry=$(find_pkgentry $pkgname) - if [ -z "$pkgentry" ]; then + if [[ -z $pkgentry ]]; then return 1 fi deltas="$pkgentry/deltas" - if [ ! -f "$deltas" ]; then + if [[ ! -f $deltas ]]; then return 1 fi if grep -q "$filename" $deltas; then @@ -183,7 +192,7 @@ db_write_entry() { # blank out all variables local pkgfile="$1" - local pkgname pkgver pkgdesc csize size md5sum url arch builddate packager force \ + local pkgname pkgver pkgdesc csize size md5sum url arch builddate packager \ _groups _licenses _replaces _depends _conflicts _provides _optdepends local OLDIFS="$IFS" @@ -213,19 +222,27 @@ db_write_entry() IFS=$OLDIFS # get md5sum and compressed size of package - md5sum="$(openssl dgst -md5 "$pkgfile" | awk '{print $NF}')" + md5sum="$(openssl dgst -md5 "$pkgfile")" + md5sum="${md5sum##* }" csize=$(stat -L -c %s "$pkgfile") # ensure $pkgname and $pkgver variables were found - if [ -z "$pkgname" -o -z "$pkgver" ]; then + if [[ -z $pkgname || -z $pkgver ]]; then error "$(gettext "Invalid package file '%s'.")" "$pkgfile" return 1 fi - cd "$tmpdir" - - if [ -d "$pkgname-$pkgver" ]; then + pushd "$tmpdir" >/dev/null + if [[ -d $pkgname-$pkgver ]]; then warning "$(gettext "An entry for '%s' already existed")" "$pkgname-$pkgver" + else + if (( DELTA )); then + pkgentry=$(find_pkgentry $pkgname) + if [[ -n $pkgentry ]]; then + local oldfilename=$(grep -A1 FILENAME $pkgentry/desc | tail -n1) + local oldfile="$(dirname $1)/$oldfilename" + fi + fi fi # remove an existing entry if it exists, ignore failures @@ -233,36 +250,35 @@ db_write_entry() # create package directory mkdir "$pkgname-$pkgver" - cd "$pkgname-$pkgver" + pushd "$pkgname-$pkgver" >/dev/null # restore an eventual deltas file - [ -f "../$pkgname.deltas" ] && mv "../$pkgname.deltas" deltas + [[ -f ../$pkgname.deltas ]] && mv "../$pkgname.deltas" deltas # create desc entry - msg2 "$(gettext "Creating 'desc' db entry...")" + msg2 "$(gettext "Creating '%s' db entry...")" 'desc' echo -e "%FILENAME%\n$(basename "$1")\n" >>desc echo -e "%NAME%\n$pkgname\n" >>desc - [ -n "$pkgbase" ] && echo -e "%BASE%\n$pkgbase\n" >>desc + [[ -n $pkgbase ]] && echo -e "%BASE%\n$pkgbase\n" >>desc echo -e "%VERSION%\n$pkgver\n" >>desc - [ -n "$pkgdesc" ] && echo -e "%DESC%\n$pkgdesc\n" >>desc + [[ -n $pkgdesc ]] && echo -e "%DESC%\n$pkgdesc\n" >>desc write_list_entry "GROUPS" "$_groups" "desc" - [ -n "$csize" ] && echo -e "%CSIZE%\n$csize\n" >>desc - [ -n "$size" ] && echo -e "%ISIZE%\n$size\n" >>desc + [[ -n $csize ]] && echo -e "%CSIZE%\n$csize\n" >>desc + [[ -n $size ]] && echo -e "%ISIZE%\n$size\n" >>desc # compute checksums msg2 "$(gettext "Computing md5 checksums...")" echo -e "%MD5SUM%\n$md5sum\n" >>desc - [ -n "$url" ] && echo -e "%URL%\n$url\n" >>desc + [[ -n $url ]] && echo -e "%URL%\n$url\n" >>desc write_list_entry "LICENSE" "$_licenses" "desc" - [ -n "$arch" ] && echo -e "%ARCH%\n$arch\n" >>desc - [ -n "$builddate" ] && echo -e "%BUILDDATE%\n$builddate\n" >>desc - [ -n "$packager" ] && echo -e "%PACKAGER%\n$packager\n" >>desc + [[ -n $arch ]] && echo -e "%ARCH%\n$arch\n" >>desc + [[ -n $builddate ]] && echo -e "%BUILDDATE%\n$builddate\n" >>desc + [[ -n $packager ]] && echo -e "%PACKAGER%\n$packager\n" >>desc write_list_entry "REPLACES" "$_replaces" "desc" - [ -n "$force" ] && echo -e "%FORCE%\n" >>desc # create depends entry - msg2 "$(gettext "Creating 'depends' db entry...")" + msg2 "$(gettext "Creating '%s' db entry...")" 'depends' # create the file even if it will remain empty touch "depends" write_list_entry "DEPENDS" "$_depends" "depends" @@ -270,9 +286,48 @@ db_write_entry() write_list_entry "PROVIDES" "$_provides" "depends" write_list_entry "OPTDEPENDS" "$_optdepends" "depends" - cd "$startdir" + popd >/dev/null + popd >/dev/null - return 0 + # create files file if wanted + if (( WITHFILES )); then + msg2 "$(gettext "Creating '%s' db entry...")" 'files' + local files_path="$tmpdir/$pkgname-$pkgver/files" + echo "%FILES%" >$files_path + bsdtar --exclude='.*' -tf "$pkgfile" >>$files_path + fi + + # create a delta file + if (( DELTA )); then + if [[ -n $oldfilename ]]; then + if [[ -f $oldfile ]]; then + delta=$(pkgdelta -q $oldfile $1) + if [[ -f $delta ]]; then + db_write_delta $delta + fi + else + warning "$(gettext "Old package file not found: %s")" "$oldfilename" + fi + fi + fi + + # Extracts licenses to a common license dir + msg "Extracting license" + if bsdtar -xOf ${pkgfile} .PKGINFO | grep "license" | grep "custom" ; then + if [ -d ${LICENSESDIR}/${pkgname} ]; then + rm -r ${LICENSESDIR}/${pkgname} + fi + + # Change dir to licenses, and extract them stripping the first part of the path + bsdtar -C ${LICENSESDIR}/ --include="usr/share/licenses/" \ + --strip-components 3 -xf ${pkgfile} >/dev/null 2>&1 + + if [ $? -ne 0 ]; then + warning "This package doesn't contain a license dir" + fi + fi + + return 0 } # end db_write_entry # remove existing entries from the DB @@ -281,9 +336,9 @@ db_remove_entry() { local pkgname=$1 local notfound=1 local pkgentry=$(find_pkgentry $pkgname) - while [ -n "$pkgentry" ]; do + while [[ -n $pkgentry ]]; do notfound=0 - if [ -f "$pkgentry/deltas" ]; then + if [[ -f $pkgentry/deltas ]]; then mv "$pkgentry/deltas" "$tmpdir/$pkgname.deltas" fi msg2 "$(gettext "Removing existing entry '%s'...")" \ @@ -291,6 +346,12 @@ db_remove_entry() { rm -rf $pkgentry pkgentry=$(find_pkgentry $pkgname) done + + msg "Removing license" + if [ -d ${LICENSESDIR}/${pkgname} ]; then + rm -r ${LICENSESDIR}/${pkgname} + fi + return $notfound } # end db_remove_entry @@ -301,16 +362,16 @@ check_repo_db() CLEAN_LOCK=1 else error "$(gettext "Failed to acquire lockfile: %s.")" "$LOCKFILE" - [ -f "$LOCKFILE" ] && error "$(gettext "Held by process %s")" "$(cat $LOCKFILE)" + [[ -f $LOCKFILE ]] && error "$(gettext "Held by process %s")" "$(cat $LOCKFILE)" exit 1 fi - if [ -f "$REPO_DB_FILE" ]; then + if [[ -f $REPO_DB_FILE ]]; then # there are two situations we can have here- a DB with some entries, # or a DB with no contents at all. if ! bsdtar -tqf "$REPO_DB_FILE" '*/desc' >/dev/null 2>&1; then # check empty case - if [ -n "$(bsdtar -tqf "$REPO_DB_FILE" '*' 2>/dev/null)" ]; then + if [[ -n $(bsdtar -tqf "$REPO_DB_FILE" '*' 2>/dev/null) ]]; then error "$(gettext "Repository file '%s' is not a proper pacman database.")" "$REPO_DB_FILE" exit 1 fi @@ -337,15 +398,15 @@ check_repo_db() add() { - if [ ! -f "$1" ]; then + if [[ ! -f $1 ]]; then error "$(gettext "File '%s' not found.")" "$1" return 1 fi - if [ "${1##*.}" == "delta" ]; then + if [[ ${1##*.} == "delta" ]]; then deltafile=$1 msg "$(gettext "Adding delta '%s'")" "$deltafile" - if [ ! "$(type -p xdelta3)" ]; then + if ! type xdelta3 &>/dev/null; then error "$(gettext "Cannot find the xdelta3 binary! Is xdelta3 installed?")" exit 1 fi @@ -357,7 +418,7 @@ add() fi pkgfile=$1 - if ! bsdtar -tqf "$pkgfile" .PKGINFO 2>&1 >/dev/null; then + if ! bsdtar -tqf "$pkgfile" .PKGINFO >/dev/null 2>&1; then error "$(gettext "'%s' is not a package file, skipping")" "$pkgfile" return 1 fi @@ -369,7 +430,7 @@ add() remove() { - if [ "${1##*.}" == "delta" ]; then + if [[ ${1##*.} == "delta" ]]; then deltafile=$1 msg "$(gettext "Searching for delta '%s'...")" "$deltafile" if db_remove_delta "$deltafile"; then @@ -402,9 +463,8 @@ trap_exit() clean_up() { local exit_code=$? - cd "$startdir" - [ -d "$tmpdir" ] && rm -rf "$tmpdir" - [ $CLEAN_LOCK -eq 1 -a -f "$LOCKFILE" ] && rm -f "$LOCKFILE" + [[ -d $tmpdir ]] && rm -rf "$tmpdir" + (( CLEAN_LOCK )) && [[ -f $LOCKFILE ]] && rm -f "$LOCKFILE" exit $exit_code } @@ -412,7 +472,7 @@ clean_up() { # PROGRAM START # determine whether we have gettext; make it a no-op if we do not -if [ ! $(type -t gettext) ]; then +if ! type gettext &>/dev/null; then gettext() { echo "$@" } @@ -423,15 +483,9 @@ case "$1" in -V|--version) version; exit 0;; esac -# check for correct number of args -if [ $# -lt 2 ]; then - usage - exit 1 -fi - # figure out what program we are cmd="$(basename $0)" -if [ "$cmd" != "repo-add" -a "$cmd" != "repo-remove" ]; then +if [[ $cmd != "repo-add" && $cmd != "repo-remove" ]]; then error "$(gettext "Invalid command name '%s' specified.")" "$cmd" exit 1 fi @@ -450,14 +504,10 @@ success=0 for arg in "$@"; do case "$arg" in -q|--quiet) QUIET=1;; - - -f|--force) - warning "$(gettext "the -f and --force options are no longer recognized")" - msg2 "$(gettext "use options=(force) in the PKGBUILD instead")" - ;; - + -d|--delta) DELTA=1;; + -f|--files) WITHFILES=1;; *) - if [ -z "$REPO_DB_FILE" ]; then + if [[ -z $REPO_DB_FILE ]]; then REPO_DB_FILE="$arg" LOCKFILE="$REPO_DB_FILE.lck" check_repo_db @@ -472,7 +522,7 @@ for arg in "$@"; do done # if at least one operation was a success, re-zip database -if [ $success -eq 1 ]; then +if (( success )); then msg "$(gettext "Creating updated database file '%s'")" "$REPO_DB_FILE" case "$REPO_DB_FILE" in @@ -485,20 +535,26 @@ if [ $success -eq 1 ]; then filename=$(basename "$REPO_DB_FILE") - cd "$tmpdir" - if [ -n "$(ls)" ]; then + pushd "$tmpdir" >/dev/null + if [[ -n $(ls) ]]; then bsdtar -c${TAR_OPT}f "$filename" * else # we have no packages remaining? zip up some emptyness warning "$(gettext "No packages remain, creating empty database.")" bsdtar -c${TAR_OPT}f "$filename" -T /dev/null fi - cd "$startdir" - - [ -f "$REPO_DB_FILE" ] && mv -f "$REPO_DB_FILE" "${REPO_DB_FILE}.old" - [ -f "$tmpdir/$filename" ] && mv "$tmpdir/$filename" "$REPO_DB_FILE" + popd >/dev/null + + [[ -f $REPO_DB_FILE ]] && mv -f "$REPO_DB_FILE" "${REPO_DB_FILE}.old" + [[ -f $tmpdir/$filename ]] && mv "$tmpdir/$filename" "$REPO_DB_FILE" + dblink="${REPO_DB_FILE%.tar.*}" + target=${REPO_DB_FILE##*/} + ln -sf "$target" "$dblink" 2>/dev/null || \ + ln -f "$target" "$dblink" 2>/dev/null || \ + cp "$REPO_DB_FILE" "$dblink" else msg "$(gettext "No packages modified, nothing to do.")" + exit 1 fi exit 0 diff --git a/repo-update b/repo-update index a44ae87..e758cc8 100755 --- a/repo-update +++ b/repo-update @@ -10,23 +10,27 @@ for repo in ${ARCHREPOS[@]}; do for arch in ${ARCHARCHES[@]} 'any'; do msg2 "${repo} ${arch}" # makes a file containing rsync output for filter.py + plain "Checking packages..." ${rsync_list_command} \ rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ - ${repodir}/staging/${repo}/ > ${rsout_file} + ${repodir}/staging/${repo}/ > ${rsout_file} 2&>/dev/null # reads blacklist and rsout_file and makes an rsync exclude-from # list + plain "Excluding nonfree..." filter.py -r ${rsync_blacklist} -k ${blacklist} \ -f ${rsout_file} # list files in ${repodir}/${repo} and write their names on # rsync_not_needed for using as an rsync exclude-from + plain "Excluding our packages" find ${repodir}/${repo} -name "*${PKGEXT}" \ -fprintf ${rsync_not_needed} '%f\n' # Actual rsync command + plain "Syncing..." ${rsync_update_command} \ --exclude-from=${rsync_blacklist} \ --exclude-from=${rsync_not_needed} \ rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ - ${repodir}/staging/${repo}/ + ${repodir}/staging/${repo}/ 2&>/dev/null done for arch in ${ARCHARCHES[@]}; do msg2 "Making pending list for $repo $arch" -- cgit v1.2.3-2-g168b From dead95b06c2a547f3cfae6c5acc21859cdd30d97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 10 Jun 2011 16:33:28 -0500 Subject: * Use python for identify nonfree in db-check-nonfree --- db-check-nonfree | 11 +++-------- list_nonfree_in_db.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 8 deletions(-) create mode 100755 list_nonfree_in_db.py diff --git a/db-check-nonfree b/db-check-nonfree index 83efb14..d8f34f0 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -23,15 +23,10 @@ for repo in ${ARCHREPOS[@]}; do if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then continue fi - unset dbpkgs unset cleanpkgs - cleanpkgs=() - dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) - for pkgname in ${dbpkgs[@]}; do - if in_array ${pkgname} ${nonfree[@]}; then - cleanpkgs[${#cleanpkgs[*]}]=${pkgname} - fi - done + cmd_="$(dirname $0)/list_nonfree_in_db.py -k ${BLACKLIST_FILE} \ + -b ${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" + cleanpkgs=($(${cmd_})) if [ ${#cleanpkgs[@]} -ge 1 ]; then arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]} fi diff --git a/list_nonfree_in_db.py b/list_nonfree_in_db.py new file mode 100755 index 0000000..598a2e7 --- /dev/null +++ b/list_nonfree_in_db.py @@ -0,0 +1,28 @@ +#! /usr/bin/python +#-*- encoding: utf-8 -*- +from repm.filter import * +import argparse + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="nonfree_in_db", + description="Cleans nonfree files on repo",) + + parser.add_argument("-k", "--blacklist-file", type=str, + help="File containing blacklisted names", + required=True,) + + parser.add_argument("-b", "--database", type=str, + help="dabatase to clean", + required=True,) + + args=parser.parse_args() + + if not (args.blacklist_file and args.database): + parser.print_help() + exit(1) + + blacklist=listado(args.blacklist_file) + pkgs=get_pkginfo_from_db(args.database) + + print(" ".join([pkg["name"] for pkg in pkgs if pkg["name"] in blacklist])) -- cgit v1.2.3-2-g168b From cd93439c7eb71c380ab0355306a760b7bb5f77ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 13 Jun 2011 06:00:17 -0500 Subject: * repo-update exclude also from ${repodir}/old/packages --- repo-update | 3 +++ 1 file changed, 3 insertions(+) diff --git a/repo-update b/repo-update index e758cc8..4631627 100755 --- a/repo-update +++ b/repo-update @@ -24,11 +24,14 @@ for repo in ${ARCHREPOS[@]}; do plain "Excluding our packages" find ${repodir}/${repo} -name "*${PKGEXT}" \ -fprintf ${rsync_not_needed} '%f\n' + find ${repodir}/old/packages -name "*${PKGEXT}" \ + -fprintf ${rsync_not_needed}2 '%f\n' # Actual rsync command plain "Syncing..." ${rsync_update_command} \ --exclude-from=${rsync_blacklist} \ --exclude-from=${rsync_not_needed} \ + --exclude-from=${rsync_not_needed}2 \ rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ ${repodir}/staging/${repo}/ 2&>/dev/null done -- cgit v1.2.3-2-g168b From c991efcdf97748614c99dc23bf8f1ecbab99a247 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 13 Jun 2011 08:37:38 -0500 Subject: fix issue 53 --- .gitignore | 5 +++-- yf-update | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100755 yf-update diff --git a/.gitignore b/.gitignore index 45688ab..ee8bbb8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,6 @@ local_config /config.local test/packages/*/*.pkg.tar.?z -#*# -.#* \ No newline at end of file +\#*# +.#* +yftime \ No newline at end of file diff --git a/yf-update b/yf-update new file mode 100755 index 0000000..c65fc09 --- /dev/null +++ b/yf-update @@ -0,0 +1,17 @@ +#!/bin/bash +source $(dirname $0)/local_config +source $(dirname $0)/config + +blacklist_mtime=$(printf "%.0f" $(find ${blacklist} -printf "%T@")) +last_bl_mtime=$(cat $(dirname $0)/yftime) + +if [ $blacklist_mtime -gt $last_bl_mtime ]; then + pushd $(dirname $0)/yf + makepkg + rsync -L ./*.${PKGEXT} ${STAGING}/libre + popd + echo ${blacklist_mtime} > $(dirname $0)/yftime +fi + + + -- cgit v1.2.3-2-g168b From 436fcd89bf7721f54a359842b4741bc67825411c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 13 Jun 2011 08:46:25 -0500 Subject: Added your-freeedom to db-scripts --- yf/PKGBUILD | 28 ++++++++++++++++++++++++++++ yf/your-freedom.install | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 yf/PKGBUILD create mode 100644 yf/your-freedom.install diff --git a/yf/PKGBUILD b/yf/PKGBUILD new file mode 100644 index 0000000..cc6f07b --- /dev/null +++ b/yf/PKGBUILD @@ -0,0 +1,28 @@ +# Maintainer: Parabola Project +pkgname=your-freedom +pkgver=$(LC_ALL=C date -u +%Y%m%d) +pkgrel=1 +pkgdesc="This package conflicts with every unfree package known to date." +arch=('any') +url="https://parabolagnulinux.org" +license=('GPL') +groups=('base') +install=${pkgname}.install +source=() +md5sums=() +noextract=() + +build() { + cd ${srcdir} + source ~/repm/local_config + install -d ${pkgdir}/usr/share/doc/${pkgname} + install -m644 $blacklist $whitelist ${pkgdir}/usr/share/doc/${pkgname}/ +} + +package() { + conflicts=($(cut -d: -f1,2 ${pkgdir}/usr/share/doc/${pkgname}/blacklist.txt | \ + sed "s/:$//" | \ + grep -v ":" | \ + sort -u + )) +} diff --git a/yf/your-freedom.install b/yf/your-freedom.install new file mode 100644 index 0000000..49ae045 --- /dev/null +++ b/yf/your-freedom.install @@ -0,0 +1,32 @@ + +pre_install() { + cat < Date: Mon, 13 Jun 2011 08:12:18 -0700 Subject: * Fixed issue131 * Fixed issue53 --- .gitignore | 4 +++- repo-update | 10 ++++++++-- yf-update | 11 ++++++----- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index ee8bbb8..dd17455 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ local_config test/packages/*/*.pkg.tar.?z \#*# .#* -yftime \ No newline at end of file +yftime +src* +pkg* \ No newline at end of file diff --git a/repo-update b/repo-update index 4631627..12c6d0c 100755 --- a/repo-update +++ b/repo-update @@ -5,6 +5,14 @@ source $(dirname $0)/config source $(dirname $0)/local_config source $(dirname $0)/libremessages +msg "Updating your-freedom" +$(dirname $0)/yf-update + +msg "Global exclude-list" +# To not downgrade repos by accident +find ${repodir}/old/packages -name "*${PKGEXT}" \ + -fprintf ${rsync_not_needed}2 '%f\n' + for repo in ${ARCHREPOS[@]}; do msg "Syncing ${repo}" for arch in ${ARCHARCHES[@]} 'any'; do @@ -24,8 +32,6 @@ for repo in ${ARCHREPOS[@]}; do plain "Excluding our packages" find ${repodir}/${repo} -name "*${PKGEXT}" \ -fprintf ${rsync_not_needed} '%f\n' - find ${repodir}/old/packages -name "*${PKGEXT}" \ - -fprintf ${rsync_not_needed}2 '%f\n' # Actual rsync command plain "Syncing..." ${rsync_update_command} \ diff --git a/yf-update b/yf-update index c65fc09..9c2131e 100755 --- a/yf-update +++ b/yf-update @@ -1,17 +1,18 @@ #!/bin/bash source $(dirname $0)/local_config source $(dirname $0)/config +source $(dirname $0)/libremessages blacklist_mtime=$(printf "%.0f" $(find ${blacklist} -printf "%T@")) last_bl_mtime=$(cat $(dirname $0)/yftime) if [ $blacklist_mtime -gt $last_bl_mtime ]; then pushd $(dirname $0)/yf - makepkg - rsync -L ./*.${PKGEXT} ${STAGING}/libre + makepkg -f + find . -name "*${PKGEXT}" -exec mv {} ${STAGING}/libre \; popd echo ${blacklist_mtime} > $(dirname $0)/yftime + msg2 "built and staged" +else + msg2 "nothing to do" fi - - - -- cgit v1.2.3-2-g168b From 7e8840e0a76d28c27ed7efc041106d338ca0c607 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Mon, 13 Jun 2011 08:17:13 -0700 Subject: repos are working fine like this --- db-check-nonfree | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/db-check-nonfree b/db-check-nonfree index d8f34f0..83efb14 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -23,10 +23,15 @@ for repo in ${ARCHREPOS[@]}; do if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then continue fi + unset dbpkgs unset cleanpkgs - cmd_="$(dirname $0)/list_nonfree_in_db.py -k ${BLACKLIST_FILE} \ - -b ${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" - cleanpkgs=($(${cmd_})) + cleanpkgs=() + dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) + for pkgname in ${dbpkgs[@]}; do + if in_array ${pkgname} ${nonfree[@]}; then + cleanpkgs[${#cleanpkgs[*]}]=${pkgname} + fi + done if [ ${#cleanpkgs[@]} -ge 1 ]; then arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]} fi -- cgit v1.2.3-2-g168b From 8a874d242c8d86059261c098dfec1b0b1ac9e444 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Mon, 13 Jun 2011 15:33:16 -0300 Subject: mkrepo is script to create [repos] --- mkrepo | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100755 mkrepo diff --git a/mkrepo b/mkrepo new file mode 100755 index 0000000..5f704cc --- /dev/null +++ b/mkrepo @@ -0,0 +1,24 @@ +#!/bin/bash +# Author: Nicolás Reynolds +# License: GPLv3+ +# Description: A script to quickly create new [repos] + +source $(dirname $0)/config +source $(dirname $0)/local_config + +# TODO it would be simpler to expand arrays to {element1,element2,etc} +for repo in $@; do + + echo ":: Creating [$repo]" + mkdir -pv ${repodir}/{staging/,}${repo} + + for arch in ${ARCHES[@]}; do + mkdir -pv ${repodir}/${repo}/os/${arch} + done + +done + +echo ":: All done. Add the repo to the parabolaweb admin page" +echo " and the get_repos script on the same server." + +exit $? -- cgit v1.2.3-2-g168b From 9bcf313647a3dd4ae44016894b1fe599d1f90c18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Sun, 19 Jun 2011 11:01:07 -0700 Subject: Config file updated, use old repo-update --- config | 4 ++-- repo-update | 17 ++--------------- 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/config b/config index effdde4..bc2494f 100644 --- a/config +++ b/config @@ -4,7 +4,7 @@ ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" ARCHREPOS=('core' 'extra' 'community' 'testing' 'multilib') -PKGREPOS=(${ARCHREPOS[@]} 'libre' 'libre-testing' 'social' 'sugar') +PKGREPOS=(${ARCHREPOS[@]} 'libre' 'libre-testing' 'social' 'sugar' 'elementary' '~fauno' '~xihh') PKGPOOL='pool/packages' SRCPOOL='sources/packages' @@ -33,4 +33,4 @@ PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" MAKEPKGCONF="$HOME/etc/makepkg.conf" -BLACKLIST_FILE="$HOME/parabolagnulinux.org/docs/blacklist.txt" \ No newline at end of file +BLACKLIST_FILE="$HOME/parabolagnulinux.org/docs/blacklist.txt" diff --git a/repo-update b/repo-update index 12c6d0c..a44ae87 100755 --- a/repo-update +++ b/repo-update @@ -5,41 +5,28 @@ source $(dirname $0)/config source $(dirname $0)/local_config source $(dirname $0)/libremessages -msg "Updating your-freedom" -$(dirname $0)/yf-update - -msg "Global exclude-list" -# To not downgrade repos by accident -find ${repodir}/old/packages -name "*${PKGEXT}" \ - -fprintf ${rsync_not_needed}2 '%f\n' - for repo in ${ARCHREPOS[@]}; do msg "Syncing ${repo}" for arch in ${ARCHARCHES[@]} 'any'; do msg2 "${repo} ${arch}" # makes a file containing rsync output for filter.py - plain "Checking packages..." ${rsync_list_command} \ rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ - ${repodir}/staging/${repo}/ > ${rsout_file} 2&>/dev/null + ${repodir}/staging/${repo}/ > ${rsout_file} # reads blacklist and rsout_file and makes an rsync exclude-from # list - plain "Excluding nonfree..." filter.py -r ${rsync_blacklist} -k ${blacklist} \ -f ${rsout_file} # list files in ${repodir}/${repo} and write their names on # rsync_not_needed for using as an rsync exclude-from - plain "Excluding our packages" find ${repodir}/${repo} -name "*${PKGEXT}" \ -fprintf ${rsync_not_needed} '%f\n' # Actual rsync command - plain "Syncing..." ${rsync_update_command} \ --exclude-from=${rsync_blacklist} \ --exclude-from=${rsync_not_needed} \ - --exclude-from=${rsync_not_needed}2 \ rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ - ${repodir}/staging/${repo}/ 2&>/dev/null + ${repodir}/staging/${repo}/ done for arch in ${ARCHARCHES[@]}; do msg2 "Making pending list for $repo $arch" -- cgit v1.2.3-2-g168b From 91210f8ae18f5651ace2273cfa0c8ad7e4dbcbdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sat, 16 Jul 2011 03:35:37 -0300 Subject: Syncs ABSLibre tarballs between ABSLibre host and repo server --- cron-jobs/update-abs-tarballs | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 cron-jobs/update-abs-tarballs diff --git a/cron-jobs/update-abs-tarballs b/cron-jobs/update-abs-tarballs new file mode 100644 index 0000000..9990f20 --- /dev/null +++ b/cron-jobs/update-abs-tarballs @@ -0,0 +1,7 @@ +#!/bin/bash + +. "$(dirname $0)/../config" + +rsync -av parabolagnulinux.org::abstar/ ${FTP_BASE}/ + +exit $? -- cgit v1.2.3-2-g168b From d438ff5d25372ba32e42f89c808f5b33a25042f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 3 Aug 2011 16:13:56 -0300 Subject: Changed SVN to ABSLibre --- db-remove | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/db-remove b/db-remove index b44eb33..ccfeb36 100755 --- a/db-remove +++ b/db-remove @@ -12,9 +12,6 @@ pkgbase="$1" repo="$2" arch="$3" -ftppath="$FTP_BASE/$repo/os" -svnrepo="$repo-$arch" - if ! check_repo_permission $repo; then die "You don't have permission to remove packages from ${repo}" fi @@ -30,14 +27,11 @@ for tarch in ${tarches[@]}; do done msg "Removing $pkgbase from [$repo]..." -/usr/bin/svn checkout -q "${SVNREPO}/${pkgbase}" "${WORKDIR}/svn/${pkgbase}" >/dev/null -if [ -d "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" ]; then - pkgnames=($(. "${WORKDIR}/svn/$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgname[@]})) - /usr/bin/svn rm --force -q "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" - /usr/bin/svn commit -q "${WORKDIR}/svn/$pkgbase" -m "$(basename $0): $pkgbase removed by $(id -un)" +if [ -d "${SVNREPO}/$repo/$pkgbase" ]; then + pkgnames=($(. "${SVNREPO}/$repo/$pkgbase/PKGBUILD"; echo ${pkgname[@]})) else - warning "$pkgbase not found in $svnrepo" + warning "$pkgbase not found in $repo" warning "Removing only $pkgbase from the repo" warning "If it was a split package you have to remove the others yourself!" pkgnames=($pkgbase) -- cgit v1.2.3-2-g168b From 69d7089ab3bec9953877f20b008e7b7dd99a3459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Wed, 3 Aug 2011 12:16:56 -0700 Subject: Added personal repos --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index bc2494f..eda45eb 100644 --- a/config +++ b/config @@ -4,7 +4,7 @@ ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" ARCHREPOS=('core' 'extra' 'community' 'testing' 'multilib') -PKGREPOS=(${ARCHREPOS[@]} 'libre' 'libre-testing' 'social' 'sugar' 'elementary' '~fauno' '~xihh') +PKGREPOS=(${ARCHREPOS[@]} 'libre-testing' 'libre' 'security' 'social' 'sugar' 'elementary' '~fauno' '~xihh' '~smv') PKGPOOL='pool/packages' SRCPOOL='sources/packages' -- cgit v1.2.3-2-g168b From 0e4184bc0c06eccd4484c9028c0369554d482aa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Fri, 5 Aug 2011 19:41:12 -0300 Subject: Added script for creating repos --- create-repo | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100755 create-repo diff --git a/create-repo b/create-repo new file mode 100755 index 0000000..58842c3 --- /dev/null +++ b/create-repo @@ -0,0 +1,24 @@ +#!/bin/bash +# Creates repository structure + +. "$(dirname $0)/db-functions" +. "$(dirname $0)/config" + +if [ $# -eq 0 ]; then + msg "Usage: $0 repo1 [repo2 ... repoX]" + exit 1 +fi + +msg "Creating repos..." +for _repo in $@; do + msg2 "Creating [${_repo}]" + mkdir -p "${FTP_BASE}/staging/${_repo}" || \ + error "Failed creating staging dir" + + for _arch in ${ARCHES[@]}; do + mkdir -p "${FTP_BASE}/${_repo}/os/${_arch}" || \ + error "Failed creating ${_arch} dir" + done +done + +msg "Don't forget to add them to the PKGREPOS array on $(dirname $0)/config" -- cgit v1.2.3-2-g168b From cd45a4e2e000312d86b072eef9349b9494cd6d36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Fri, 5 Aug 2011 19:49:23 -0300 Subject: Repo organization --- config | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/config b/config index effdde4..8fd37bf 100644 --- a/config +++ b/config @@ -3,8 +3,15 @@ FTP_BASE="/home/parabolavnx/parabolagnulinux.org/repo" ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" +# Repos from Arch ARCHREPOS=('core' 'extra' 'community' 'testing' 'multilib') -PKGREPOS=(${ARCHREPOS[@]} 'libre' 'libre-testing' 'social' 'sugar') +# Official Parabola repos +OURREPOS=('libre' 'libre-testing') +# User repos +USERREPOS=('~fauno' '~smv' '~xihh') +# Community project repos +PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'social') +PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) PKGPOOL='pool/packages' SRCPOOL='sources/packages' @@ -33,4 +40,4 @@ PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" MAKEPKGCONF="$HOME/etc/makepkg.conf" -BLACKLIST_FILE="$HOME/parabolagnulinux.org/docs/blacklist.txt" \ No newline at end of file +BLACKLIST_FILE="$HOME/parabolagnulinux.org/docs/blacklist.txt" -- cgit v1.2.3-2-g168b From c1858e5d04ebe2b8c2c45ff71dc8bcfb05b49dd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Tue, 9 Aug 2011 21:46:24 -0300 Subject: update-abs-tarballs +x --- cron-jobs/update-abs-tarballs | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 cron-jobs/update-abs-tarballs diff --git a/cron-jobs/update-abs-tarballs b/cron-jobs/update-abs-tarballs old mode 100644 new mode 100755 -- cgit v1.2.3-2-g168b From 67c5e5b37efa306c92644922fad65adbb127a4bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Tue, 9 Aug 2011 21:49:27 -0300 Subject: Ignore staging --- cron-jobs/update-abs-tarballs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/update-abs-tarballs b/cron-jobs/update-abs-tarballs index 9990f20..824ac34 100755 --- a/cron-jobs/update-abs-tarballs +++ b/cron-jobs/update-abs-tarballs @@ -2,6 +2,6 @@ . "$(dirname $0)/../config" -rsync -av parabolagnulinux.org::abstar/ ${FTP_BASE}/ +rsync -av --exclude=staging/ parabolagnulinux.org::abstar/ ${FTP_BASE}/ exit $? -- cgit v1.2.3-2-g168b From 986fbe51101674c290df8f5c9d0dc3f507c172e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Thu, 11 Aug 2011 20:45:29 -0300 Subject: Script to solve issue165 --- cron-jobs/repo-sanity-check | 46 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 cron-jobs/repo-sanity-check diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check new file mode 100644 index 0000000..a6249b5 --- /dev/null +++ b/cron-jobs/repo-sanity-check @@ -0,0 +1,46 @@ +#!/bin/bash +# Solves issue165 + +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" + +# Traverse all repos +for _repo in ${REPOS[@]}; do + +# Find all pkgnames on this repo's abs + on_abs=($( + find ${SVNREPO}/${_repo} -name PKGBUILD | \ + while read pkgbuild; do + source ${pkgbuild} >/dev/null 2>&1 +# cleanup to save memory + unset build package source md5sums pkgdesc pkgver pkgrel epoch \ + url license arch depends makedepends optdepends options \ + >/dev/null 2>&1 + +# also cleanup package functions + for _pkg in ${pkgname[@]}; do + unset package_${pkg} >/dev/null 2>&1 + done + +# this fills the on_abs array + echo ${pkgname[@]} + done + )) + +# Find all pkgnames on repos + on_repo=($( + find ${FTP_BASE}/${_repo} -name "*.pkg.tar.?z" -printf "%f\n" | \ + sed "s/^\(.\+\)-[^-]\+-[^-]\+-[^-]\+$/\1/" + )) + +# Compares them, whatever is on repos but not on abs should be removed + remove=($(comm -13 <(echo ${on_abs[@]} | tr ' ' "\n" | sort) \ + <(echo ${on_repo[@]} | tr ' ' "\n" | sort))) + +# Remove them from databases, ftpdir-cleanup will take care of the rest + find ${FTP_BASE}/${_repo} -name "*.db.tar.?z" -print0 | \ + xargs -0 repo-remove {} ${remove[@]} + +done + +exit $? -- cgit v1.2.3-2-g168b From d1de884ad56e0756a80d5c00bb723e7f2a56491b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Thu, 11 Aug 2011 21:37:21 -0300 Subject: Changed xargs for find -exec --- cron-jobs/repo-sanity-check | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index a6249b5..52ca33b 100644 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -38,8 +38,8 @@ for _repo in ${REPOS[@]}; do <(echo ${on_repo[@]} | tr ' ' "\n" | sort))) # Remove them from databases, ftpdir-cleanup will take care of the rest - find ${FTP_BASE}/${_repo} -name "*.db.tar.?z" -print0 | \ - xargs -0 repo-remove {} ${remove[@]} + find ${FTP_BASE}/${_repo} -name "*.db.tar.?z" -exec \ + repo-remove {} ${remove[@]} \; done -- cgit v1.2.3-2-g168b From 1d7db341d4f27f8ebd7167a8630ea8589e40c464 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Thu, 11 Aug 2011 21:39:05 -0300 Subject: Forgot +x --- cron-jobs/repo-sanity-check | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 cron-jobs/repo-sanity-check diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check old mode 100644 new mode 100755 -- cgit v1.2.3-2-g168b From d2f65c340f890050c99371e203288afea03d125b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Thu, 11 Aug 2011 22:42:01 -0300 Subject: Sort unique pkgnames --- cron-jobs/repo-sanity-check | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index 52ca33b..690a9f8 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -34,8 +34,8 @@ for _repo in ${REPOS[@]}; do )) # Compares them, whatever is on repos but not on abs should be removed - remove=($(comm -13 <(echo ${on_abs[@]} | tr ' ' "\n" | sort) \ - <(echo ${on_repo[@]} | tr ' ' "\n" | sort))) + remove=($(comm -13 <(echo ${on_abs[@]} | tr ' ' "\n" | sort -u) \ + <(echo ${on_repo[@]} | tr ' ' "\n" | sort -u))) # Remove them from databases, ftpdir-cleanup will take care of the rest find ${FTP_BASE}/${_repo} -name "*.db.tar.?z" -exec \ -- cgit v1.2.3-2-g168b From 7d6595c750903114740c7ab726cd89cb73d6c3a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Fri, 12 Aug 2011 02:37:50 -0300 Subject: Fixed unexistent REPOS array and skip removal when the ABS tree is empty. --- cron-jobs/repo-sanity-check | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index 690a9f8..92b26ae 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -5,7 +5,7 @@ . "$(dirname $0)/../config" # Traverse all repos -for _repo in ${REPOS[@]}; do +for _repo in ${PKGREPOS[@]}; do # Find all pkgnames on this repo's abs on_abs=($( @@ -27,6 +27,12 @@ for _repo in ${REPOS[@]}; do done )) +# quit if abs is empty + if [ ${#on_abs[*]} -eq 0 ]; then + warning "[${_repo}]'s ABS tree is empty, skipping" + break + fi + # Find all pkgnames on repos on_repo=($( find ${FTP_BASE}/${_repo} -name "*.pkg.tar.?z" -printf "%f\n" | \ -- cgit v1.2.3-2-g168b From a032c1fad3e2d57a7a1b7cef65d90ea06a766784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sat, 13 Aug 2011 12:42:42 -0300 Subject: Cleaner sourceballs2 --- cron-jobs/sourceballs2 | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index b29b396..a43d71e 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -33,24 +33,28 @@ for repo in ${PKGREPOS[@]}; do continue } - unset pkgbase pkgname +# Unset the previous data + unset pkgbase pkgname pkgver pkgrel source PKGBUILD + + unset build package url pkgdesc source md5sums depends makedepends \ + optdepends license arch options check mksource + + for _pkg in ${pkgname[@]}; do + unset package_${_pkg} >/dev/null 2>&1 + done + pkgbase=${pkgbase:-$pkgname} + srcfile="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" - echo "${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" + echo "${srcfile}" >> "${WORKDIR}/expected-src-pkgs" # Skip already sourceballed - [[ -e "${SRCPKGDEST}/${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" ]] && \ - continue + [ -e "${SRCPKGDEST}/${srcfile}" ] && continue - msg2 "$pkgbase-$pkgver-$pkgrel..." - makepkg --allsource --ignorearch -c makepkg --allsource --ignorearch -c >/dev/null 2>&1 - [[ $? -ne 0 ]] && { - warning "Failed." - failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" - } + [ $? -ne 0 ] && failedpkgs+=("${srcfile}") done popd >/dev/null @@ -89,10 +93,5 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then done fi -msg "Failed" -for _fail in ${failedpkgs[@]}; do - msg2 "$_fail" -done - script_unlock -- cgit v1.2.3-2-g168b From f23d7103fec259ebe2ba796ae7f351dfbda5b705 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sat, 13 Aug 2011 12:51:32 -0300 Subject: Added useful messages --- cron-jobs/repo-sanity-check | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index 92b26ae..1ba90a6 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -6,6 +6,7 @@ # Traverse all repos for _repo in ${PKGREPOS[@]}; do + msg "Cleaning up [${_repo}]" # Find all pkgnames on this repo's abs on_abs=($( @@ -45,7 +46,10 @@ for _repo in ${PKGREPOS[@]}; do # Remove them from databases, ftpdir-cleanup will take care of the rest find ${FTP_BASE}/${_repo} -name "*.db.tar.?z" -exec \ - repo-remove {} ${remove[@]} \; + repo-remove {} ${remove[@]} >/dev/null 2>&1 \; + + msg2 "Removed the following packages:" + plain "$(echo ${remove[@]} | tr ' ' "\n")" done -- cgit v1.2.3-2-g168b From eac9e4d43cc33bf6671fdbd7f04e5bab387259d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sat, 13 Aug 2011 13:15:35 -0300 Subject: Failedpkgs array wasn't working --- cron-jobs/sourceballs2 | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index a43d71e..5e228fc 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -22,10 +22,10 @@ find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort pushd "${SVNREPO}" >/dev/null for repo in ${PKGREPOS[@]}; do - failedpkgs=() + msg "Sourceballing [${repo}]" pushd $repo >/dev/null - find . -maxdepth 1 -type d | while read pkg; do + find -maxdepth 1 -type d | while read pkg; do pushd "${SVNREPO}/$repo/$pkg" >/dev/null [[ ! -e PKGBUILD ]] && { @@ -54,18 +54,12 @@ for repo in ${PKGREPOS[@]}; do makepkg --allsource --ignorearch -c >/dev/null 2>&1 - [ $? -ne 0 ] && failedpkgs+=("${srcfile}") + [ $? -ne 0 ] && plain ${srcfile} - done + done # end find pkgs popd >/dev/null - if [ ${#failedpkgs[@]} -ge 1 ]; then - msg "Failed to create source packages for [${repo}]..." - for failed_pkg in ${failedpkgs[@]}; do - msg2 "${failed_pkg}" - done - fi -done +done # end repos # Cleanup old source packages cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" -- cgit v1.2.3-2-g168b From 119de3a00044c0f854c21cd3620666414b1c6a7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 17 Aug 2011 18:14:44 -0300 Subject: Script to restore packages from the cleanup destdir --- repo-restore-to-normal | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100755 repo-restore-to-normal diff --git a/repo-restore-to-normal b/repo-restore-to-normal new file mode 100755 index 0000000..e46cfc9 --- /dev/null +++ b/repo-restore-to-normal @@ -0,0 +1,54 @@ +#!/bin/bash +# Solves issue165 + +. "$(dirname $0)/db-functions" +. "$(dirname $0)/config" + +# Find all pkgnames on old with pkgvers + on_repo=($(find ${CLEANUP_DESTDIR} -name "*.pkg.tar.?z" -printf "%f\n" | \ + sed "s/^\(.\+-[^-]\+\)-[^-]\+-[^-]\+$/\1/")) + +# Traverse all repos +for _repo in ${PKGREPOS[@]}; do + msg "Restoring [${_repo}]" + +# Find all pkgnames on this repo's abs + on_abs=($( + find ${SVNREPO}/${_repo} -name PKGBUILD | \ + while read pkgbuild; do + source ${pkgbuild} >/dev/null 2>&1 +# cleanup to save memory + unset build package source md5sums pkgdesc pkgrel epoch \ + url license arch depends makedepends optdepends options \ + >/dev/null 2>&1 + +# also cleanup package functions + for _pkg in ${pkgname[@]}; do + unset package_${pkg} >/dev/null 2>&1 + done + +# this fills the on_abs array + echo ${pkgname[@]}-${pkgver} + done + )) + +# quit if abs is empty + if [ ${#on_abs[*]} -eq 0 ]; then + warning "[${_repo}]'s ABS tree is empty, skipping" + break + fi + +# Compares them, whatever is on abs should be restored + restore=($(comm -12 <(echo ${on_abs[@]} | tr ' ' "\n" | sort -u) \ + <(echo ${on_repo[@]} | tr ' ' "\n" | sort -u))) + + msg2 "Restoring the following packages:" + plain "$(echo ${restore[@]} | tr ' ' "\n")" + + for _pkg in ${restore[@]}; do + find ${CLEANUP_DESTDIR} -name "${_pkg}*" -exec mv -v '{}' ${STAGING}/${_repo} + done + +done + +exit $? -- cgit v1.2.3-2-g168b From 4d760a1e9b92168a07aa8c7a3a9de481ebcac2a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 9 Sep 2011 12:33:51 -0700 Subject: Added [~mtjm], changed blacklist to git --- config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config b/config index 8fd37bf..0e8dc2e 100644 --- a/config +++ b/config @@ -8,7 +8,7 @@ ARCHREPOS=('core' 'extra' 'community' 'testing' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos -USERREPOS=('~fauno' '~smv' '~xihh') +USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm') # Community project repos PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'social') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) @@ -40,4 +40,4 @@ PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" MAKEPKGCONF="$HOME/etc/makepkg.conf" -BLACKLIST_FILE="$HOME/parabolagnulinux.org/docs/blacklist.txt" +BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" -- cgit v1.2.3-2-g168b From 178c720a4c2fd8adf20e6109784c39cc27dd6b30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 9 Sep 2011 12:35:46 -0700 Subject: Inform found packages --- db-check-nonfree | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db-check-nonfree b/db-check-nonfree index 83efb14..ab6491d 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -29,10 +29,11 @@ for repo in ${ARCHREPOS[@]}; do dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) for pkgname in ${dbpkgs[@]}; do if in_array ${pkgname} ${nonfree[@]}; then - cleanpkgs[${#cleanpkgs[*]}]=${pkgname} + cleanpkgs+=(${pkgname}) fi done if [ ${#cleanpkgs[@]} -ge 1 ]; then + msg2 "Unfree: ${cleanpkgs[@]}" arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]} fi done -- cgit v1.2.3-2-g168b From e9d0581b173853e647b36caa170b7c4bbee43643 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joshua=20Ismael=20Haase=20Hern=C3=A1ndez?= Date: Fri, 9 Sep 2011 12:39:56 -0700 Subject: Several fixes --- repo-restore-to-normal | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/repo-restore-to-normal b/repo-restore-to-normal index e46cfc9..9463731 100755 --- a/repo-restore-to-normal +++ b/repo-restore-to-normal @@ -4,9 +4,12 @@ . "$(dirname $0)/db-functions" . "$(dirname $0)/config" -# Find all pkgnames on old with pkgvers - on_repo=($(find ${CLEANUP_DESTDIR} -name "*.pkg.tar.?z" -printf "%f\n" | \ - sed "s/^\(.\+-[^-]\+\)-[^-]\+-[^-]\+$/\1/")) +CLEANUP_DESTDIR=/home/parabolavnx/repo/pool/restore +PKGREPOS=(community) + +# Find all pkgnames on old with pkgver-pkgrels +#on_repo=($(find ${CLEANUP_DESTDIR} -name "*.pkg.tar.?z" -printf "%f\n" | \ +# sed "s/^\(.\+-[^-]\+-[^-]\+\)-[^-]\+$/\1/")) # Traverse all repos for _repo in ${PKGREPOS[@]}; do @@ -16,37 +19,38 @@ for _repo in ${PKGREPOS[@]}; do on_abs=($( find ${SVNREPO}/${_repo} -name PKGBUILD | \ while read pkgbuild; do + unset pkgname pkgver pkgrel source ${pkgbuild} >/dev/null 2>&1 # cleanup to save memory - unset build package source md5sums pkgdesc pkgrel epoch \ + unset build package source md5sums pkgdesc epoch \ url license arch depends makedepends optdepends options \ >/dev/null 2>&1 # also cleanup package functions for _pkg in ${pkgname[@]}; do unset package_${pkg} >/dev/null 2>&1 +# this fills the on_abs array + echo ${_pkg}-${pkgver}-${pkgrel} done -# this fills the on_abs array - echo ${pkgname[@]}-${pkgver} done )) # quit if abs is empty if [ ${#on_abs[*]} -eq 0 ]; then warning "[${_repo}]'s ABS tree is empty, skipping" - break + continue fi # Compares them, whatever is on abs should be restored - restore=($(comm -12 <(echo ${on_abs[@]} | tr ' ' "\n" | sort -u) \ - <(echo ${on_repo[@]} | tr ' ' "\n" | sort -u))) +# restore=($(comm -12 <(echo ${on_abs[@]} | tr ' ' "\n" | sort -u) \ +# <(echo ${on_repo[@]} | tr ' ' "\n" | sort -u))) msg2 "Restoring the following packages:" - plain "$(echo ${restore[@]} | tr ' ' "\n")" +# plain "$(echo ${restore[@]} | tr ' ' "\n")" - for _pkg in ${restore[@]}; do - find ${CLEANUP_DESTDIR} -name "${_pkg}*" -exec mv -v '{}' ${STAGING}/${_repo} + for _pkg in ${on_abs[@]}; do + find ${CLEANUP_DESTDIR} -name "${_pkg}*" -exec cp -v '{}' ${STAGING}/${_repo} \; done done -- cgit v1.2.3-2-g168b From 756f1c3b8bb39c9932b04cea05adecb71553c982 Mon Sep 17 00:00:00 2001 From: Parabola Date: Wed, 14 Sep 2011 05:40:17 -0700 Subject: Move to python2 --- clean_repo.py | 4 ++-- config.py | 2 +- filter.py | 4 ++-- list_nonfree_in_db.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/clean_repo.py b/clean_repo.py index cc8e811..076de60 100755 --- a/clean_repo.py +++ b/clean_repo.py @@ -1,6 +1,6 @@ -#! /usr/bin/python +#! /usr/bin/python2 #-*- encoding: utf-8 -*- -from repm.filter import * +from filter import * import argparse def mkpending(packages_iterable, pending_file, blacklisted_names, diff --git a/config.py b/config.py index 4e218a5..0ffc475 100755 --- a/config.py +++ b/config.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python2 # -*- coding: utf-8 -*- try: from subprocess import check_output diff --git a/filter.py b/filter.py index 5d90bdd..70d5e29 100755 --- a/filter.py +++ b/filter.py @@ -1,7 +1,7 @@ -#! /usr/bin/python +#! /usr/bin/python2 #-*- encoding: utf-8 -*- from glob import glob -from repm.config import * +from config import * import tarfile def listado(filename, start=0, end=None): diff --git a/list_nonfree_in_db.py b/list_nonfree_in_db.py index 598a2e7..4e1b164 100755 --- a/list_nonfree_in_db.py +++ b/list_nonfree_in_db.py @@ -1,6 +1,6 @@ -#! /usr/bin/python +#! /usr/bin/python2 #-*- encoding: utf-8 -*- -from repm.filter import * +from filter import * import argparse if __name__ == "__main__": -- cgit v1.2.3-2-g168b From 4e81322477747ad61524f1d18fcba58965872542 Mon Sep 17 00:00:00 2001 From: Parabola Date: Wed, 14 Sep 2011 05:40:47 -0700 Subject: Fixed sync paths * rsync files are splitted by repo and arch * compare available packages with pool, not repos (dead symlinks broke syncing) * receive from repo/os/arch instead of repo/ (it made packages on other arches missing) --- repo-update | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/repo-update b/repo-update index a44ae87..4f31116 100755 --- a/repo-update +++ b/repo-update @@ -6,28 +6,31 @@ source $(dirname $0)/local_config source $(dirname $0)/libremessages for repo in ${ARCHREPOS[@]}; do + msg "Syncing ${repo}" - for arch in ${ARCHARCHES[@]} 'any'; do - msg2 "${repo} ${arch}" - # makes a file containing rsync output for filter.py - ${rsync_list_command} \ - rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ - ${repodir}/staging/${repo}/ > ${rsout_file} - # reads blacklist and rsout_file and makes an rsync exclude-from - # list - filter.py -r ${rsync_blacklist} -k ${blacklist} \ - -f ${rsout_file} - # list files in ${repodir}/${repo} and write their names on - # rsync_not_needed for using as an rsync exclude-from - find ${repodir}/${repo} -name "*${PKGEXT}" \ - -fprintf ${rsync_not_needed} '%f\n' - # Actual rsync command - ${rsync_update_command} \ - --exclude-from=${rsync_blacklist} \ - --exclude-from=${rsync_not_needed} \ - rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ - ${repodir}/staging/${repo}/ - done + for arch in ${ARCHARCHES[@]}; do + msg2 "${repo} ${arch}" + # makes a file containing rsync output for filter.py + ${rsync_list_command} \ + rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ + ${repodir}/staging/${repo}/os/${arch}/ > ${rsout_file}-${repo}-${arch} || exit 1 + # reads blacklist and rsout_file and makes an rsync exclude-from + # list + filter.py -r ${rsync_blacklist} -k ${blacklist} \ + -f ${rsout_file}-${repo}-${arch} || exit 1 + # list files in ${repodir}/${repo} and write their names on + # rsync_not_needed for using as an rsync exclude-from + #find ${repodir}/${repo}/os/${arch}/ -name "*${PKGEXT}" \ + # -fprintf ${rsync_not_needed}-${repo}-${arch} '%f\n' || exit 1 + find ${repodir}/${PKGPOOL}/ -name "*${PKGEXT}" \ + -fprintf ${rsync_not_needed}-${repo}-${arch} '%f\n' || exit 1 + # Actual rsync command + ${rsync_update_command} \ + --exclude-from=${rsync_blacklist} \ + --exclude-from=${rsync_not_needed}-${repo}-${arch} \ + rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ + ${repodir}/staging/${repo}/ || exit 1 + done for arch in ${ARCHARCHES[@]}; do msg2 "Making pending list for $repo $arch" # if there is a db in repo (db is created on rsync) @@ -42,7 +45,7 @@ for repo in ${ARCHREPOS[@]}; do done # if some nonfree files got pass the filter this command delete them msg2 "Fallback cleaning repo" - $(dirname $0)/clean_repo.py -k ${blacklist} -d ${repodir}/staging/${repo} + $(dirname $0)/clean_repo.py -k ${blacklist} -d ${repodir}/staging/${repo} || exit 1 done msg "Removing leftover files..." -- cgit v1.2.3-2-g168b From 9612e5d915faf63ea6d5a5ca5c3ff74cca8eb923 Mon Sep 17 00:00:00 2001 From: Nicolas Reynolds Date: Fri, 28 Oct 2011 12:15:09 -0300 Subject: Reflect server changes on config --- config | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/config b/config index effdde4..4f4d81d 100644 --- a/config +++ b/config @@ -1,7 +1,7 @@ #!/bin/bash -FTP_BASE="/home/parabolavnx/parabolagnulinux.org/repo" -ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" -SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" +FTP_BASE="/home/repo/public" +ARCH_BASE="/home/repo/public" +SVNREPO="/home/repo/abslibre" ARCHREPOS=('core' 'extra' 'community' 'testing' 'multilib') PKGREPOS=(${ARCHREPOS[@]} 'libre' 'libre-testing' 'social' 'sugar') @@ -24,7 +24,7 @@ LOCK_DELAY=10 LOCK_TIMEOUT=300 STAGING="$FTP_BASE/staging" -TMPDIR="$HOME/tmp" +TMPDIR="/tmp" ARCHARCHES=(i686 x86_64) ARCHES=(${ARCHARCHES[@]} mips64el) DBEXT=".db.tar.gz" @@ -32,5 +32,5 @@ FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" -MAKEPKGCONF="$HOME/etc/makepkg.conf" -BLACKLIST_FILE="$HOME/parabolagnulinux.org/docs/blacklist.txt" \ No newline at end of file +MAKEPKGCONF="/etc/makepkg.conf" +BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" -- cgit v1.2.3-2-g168b From 8bb0c5aa9d00dfe16101329d62691d3f60cf134b Mon Sep 17 00:00:00 2001 From: Nicolas Reynolds Date: Fri, 28 Oct 2011 12:18:29 -0300 Subject: Sync package databases first * Get all available packages * Remove unfree from the sync list * Sync everything whitelisted --- db-sync | 108 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 db-sync diff --git a/db-sync b/db-sync new file mode 100644 index 0000000..0baf497 --- /dev/null +++ b/db-sync @@ -0,0 +1,108 @@ +#!/bin/bash +# Syncs Arch repos based on info contained in repo.db files +# License: GPLv3 + +# Principles +# * Get repo.db from an Arch-like repo +# * Generate a list of available packages +# * Create sync whitelist (based on package blacklist) +# * Get packages +# * Check package signatures +# * Check database signatures +# * Sync repo => repo + + +# eval this +BASEURL="ftp://ftp.archlinux.org/\$repo/os/\$arch/\$file" + +# Generates an URL from BASE_URL +# _Params_ +# * repo +# * arch +# * file +eval_url() { + repo="$1" + arch="$2" + file="$3" + + eval "${BASE_URL}" +} + +# Returns contents of a repo +get_repos() { + rsync -av --include="*.db*" --exclude="*" rsync://${mirror}/${mirror_path}/ cache/ +} + +get_repo_content() { +# Return all contents + bsdtar tf cache/$1/os/$2/$1.db.tar.* | \ + cut -d "/" -f 1 | \ + sort -u +} + +# Get the database compression as an extension +get_repo_ext() { + file "$1" | tr A-Z a-z | sed -e "s/^[^:]\+: *\(.z\).*$/.tar.\1/" -e "s/bz/&2" +} + +# Prints blacklisted packages +get_blacklist() { + cut -d ':' -f 1 "${BLACKLIST_FILE}" +} + +# repo +# arch +get_repo_file() { + [ ! -f "cache/${1}/os/${2}/${1}.db.tar.*" ] && return 1 + + echo cache/${1}/os/${2}/${1}.db.tar.* +} + +# Process the databases and get the libre packages +init() { +# Fail on every error + set -E + + source $(dirname $0)/config + source $(dirname $0)/local_config + source $(dirname $0)/libremessages + +# Get the blacklisted packages + blacklist=($(get_blacklist)) + +# Sync the repos databases + get_repos + +# Traverse all repo-arch pairs + for _arch in ${ARCHARCHES[@]}; do + for _repo in ${ARCHREPOS[@]}; do + msg "Processing ${_repo}-${_arch}" + + repo_file=$(get_repo_file ${_repo} ${_arch}) + +# Remove blacklisted packages and count them + msg2 "Removing blacklisted packages: $( + LC_ALL=C repo-remove ${repo_file} ${blacklist[@]} 2>&1 | \ + grep "\-> Removing" 2>/dev/null| wc -l)" + +# Get db contents + db=($(get_repo ${_repo} ${_arch})) + + msg2 "Process clean db for syncing..." + +# Create a whitelist + echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist + +# Sync excluding everything but blacklist + rsync -av --include-from=/tmp/${_repo}-${_arch}.whitelist --exclude="*" + + +# Cleanup + unset db + done + done + +# Cleanup + unset blacklist _arch _repo repo_file +} + -- cgit v1.2.3-2-g168b From 666f2b153c4bd06259124dcc6641db6b21495787 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sun, 30 Oct 2011 20:34:34 -0300 Subject: Seems to work, entering test status. --- createrepos | 8 +++++ db-sync | 115 ++++++++++++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 108 insertions(+), 15 deletions(-) create mode 100755 createrepos mode change 100644 => 100755 db-sync diff --git a/createrepos b/createrepos new file mode 100755 index 0000000..4ee057b --- /dev/null +++ b/createrepos @@ -0,0 +1,8 @@ +#!/bin/bash +# Creates the repo structure defined in config + +source $(dirname $0)/config + +mkdir -p ${FTP_BASE}/{${PKGPOOL},${SRCPOOL}} ${ARCH_BASE} ${CLEANUP_DESTDIR} ${SOURCE_CLEANUP_DESTDIR} ${STAGING} + +$(dirname $0)/create-repo ${PKGREPOS[@]} diff --git a/db-sync b/db-sync old mode 100644 new mode 100755 index 0baf497..4c5dd7a --- a/db-sync +++ b/db-sync @@ -11,8 +11,13 @@ # * Check database signatures # * Sync repo => repo +# TODO +# * verbose mode +# * make a tarball of files used for forensics +# * get files db # eval this +# *not needed* BASEURL="ftp://ftp.archlinux.org/\$repo/os/\$arch/\$file" # Generates an URL from BASE_URL @@ -30,12 +35,18 @@ eval_url() { # Returns contents of a repo get_repos() { - rsync -av --include="*.db*" --exclude="*" rsync://${mirror}/${mirror_path}/ cache/ +# Exclude everything but db files + rsync -avm --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --exclude="*" \ + --delete-after \ + rsync://${mirror}/${mirrorpath}/ cache/ } get_repo_content() { # Return all contents - bsdtar tf cache/$1/os/$2/$1.db.tar.* | \ + bsdtar tf ${1} | \ cut -d "/" -f 1 | \ sort -u } @@ -53,22 +64,20 @@ get_blacklist() { # repo # arch get_repo_file() { - [ ! -f "cache/${1}/os/${2}/${1}.db.tar.*" ] && return 1 + [ ! -r "cache/${1}/os/${2}/${1}${DBEXT}" ] && return 1 - echo cache/${1}/os/${2}/${1}.db.tar.* + echo "cache/${1}/os/${2}/${1}${DBEXT}" } # Process the databases and get the libre packages init() { -# Fail on every error - set -E - - source $(dirname $0)/config - source $(dirname $0)/local_config - source $(dirname $0)/libremessages # Get the blacklisted packages blacklist=($(get_blacklist)) +# Store all the whitelist files + whitelists=() + + msg "${#blacklist[@]} packages in blacklist" # Sync the repos databases get_repos @@ -81,28 +90,104 @@ init() { repo_file=$(get_repo_file ${_repo} ${_arch}) # Remove blacklisted packages and count them +# TODO capture all removed packages for printing on debug mode msg2 "Removing blacklisted packages: $( LC_ALL=C repo-remove ${repo_file} ${blacklist[@]} 2>&1 | \ grep "\-> Removing" 2>/dev/null| wc -l)" # Get db contents - db=($(get_repo ${_repo} ${_arch})) + db=($(get_repo_content ${repo_file})) msg2 "Process clean db for syncing..." -# Create a whitelist +# Create a whitelist, add * wildcard to end +# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if +# we aren't syncing them echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist -# Sync excluding everything but blacklist - rsync -av --include-from=/tmp/${_repo}-${_arch}.whitelist --exclude="*" + msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" + +# Sync excluding everything but whitelist +# We delete here for cleanup + rsync -vrtlH \ + --delete-after \ + --safe-links \ + --delay-updates \ + --max-delete=1000 \ + --include-from=/tmp/${_repo}-${_arch}.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ +# Add a new whitelist + whitelists+=(/tmp/${_repo}-${_arch}.whitelist) # Cleanup unset db done done + msg "Putting databases back in place" + rsync -vrtlH \ + --delay-updates \ + --safe-links \ + cache/ \ + ${FTP_BASE}/ + + msg "Syncing package pool" +# Concatenate all whitelists + cat ${whitelists[@]} | sort -u > /tmp/any.whitelist + + msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" + +# Sync +# *Don't delete-after*, this is the job of cleanup scripts. It will remove our +# packages too + rsync -vrtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ + ${FTP_BASE}/${PKGPOOL}/ + +# Sync sources + msg "Syncing source pool" + #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist + + #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool" +# Sync +# *Don't delete-after*, this is the job of cleanup scripts. It will remove our +# packages too + rsync -vrtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ + ${FTP_BASE}/${SRCPOOL}/ + + # Cleanup - unset blacklist _arch _repo repo_file + unset blacklist whitelists _arch _repo repo_file +} + +trap_exit() { + echo + error "$@" + exit 1 } + +source $(dirname $0)/config +source $(dirname $0)/local_config +source $(dirname $0)/libremessages + +# From makepkg +set -E + +trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT +trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT +trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR + +init -- cgit v1.2.3-2-g168b From 92a8525eb5ed349f95c080c61c821399e3917842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sun, 30 Oct 2011 20:49:41 -0300 Subject: Local test config --- config | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/config b/config index c8b3f0b..9a20f4a 100644 --- a/config +++ b/config @@ -1,10 +1,10 @@ #!/bin/bash -FTP_BASE="/home/repo/public" -ARCH_BASE="/home/repo/public" -SVNREPO="/home/repo/abslibre" +FTP_BASE="/tmp/repo" +ARCH_BASE="/tmp/repo" +SVNREPO="/var/abs" # Repos from Arch -ARCHREPOS=('core' 'extra' 'community' 'testing' 'multilib') +ARCHREPOS=('core') #'extra' 'community' 'testing' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos @@ -32,7 +32,7 @@ LOCK_TIMEOUT=300 STAGING="$FTP_BASE/staging" TMPDIR="/tmp" -ARCHARCHES=(i686 x86_64) +ARCHARCHES=(i686) # x86_64) ARCHES=(${ARCHARCHES[@]} mips64el) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" @@ -40,4 +40,4 @@ PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" MAKEPKGCONF="/etc/makepkg.conf" -BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" +BLACKLIST_FILE="$FTP_BASE/blacklist.txt" -- cgit v1.2.3-2-g168b From c60d06e050e929d454b447a3cac3263a83d0a445 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Thu, 3 Nov 2011 15:42:12 -0300 Subject: Removed unneeded code --- db-sync | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/db-sync b/db-sync index 4c5dd7a..0bb79b7 100755 --- a/db-sync +++ b/db-sync @@ -16,23 +16,6 @@ # * make a tarball of files used for forensics # * get files db -# eval this -# *not needed* -BASEURL="ftp://ftp.archlinux.org/\$repo/os/\$arch/\$file" - -# Generates an URL from BASE_URL -# _Params_ -# * repo -# * arch -# * file -eval_url() { - repo="$1" - arch="$2" - file="$3" - - eval "${BASE_URL}" -} - # Returns contents of a repo get_repos() { # Exclude everything but db files @@ -51,11 +34,6 @@ get_repo_content() { sort -u } -# Get the database compression as an extension -get_repo_ext() { - file "$1" | tr A-Z a-z | sed -e "s/^[^:]\+: *\(.z\).*$/.tar.\1/" -e "s/bz/&2" -} - # Prints blacklisted packages get_blacklist() { cut -d ':' -f 1 "${BLACKLIST_FILE}" -- cgit v1.2.3-2-g168b From 801ea2c927ace5ee892209dd8e3c1044e1b3842e Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 14 Nov 2011 05:14:04 -0800 Subject: Fixes while testing it --- config | 10 +++++----- db-sync | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config b/config index a0849a9..23b7bbb 100644 --- a/config +++ b/config @@ -1,10 +1,10 @@ #!/bin/bash -FTP_BASE="/tmp/repo" -ARCH_BASE="/tmp/repo" +FTP_BASE="/srv/http/repo/public/temprepo" +ARCH_BASE="/srv/http/repo/public/temprepo" SVNREPO="/var/abs" # Repos from Arch -ARCHREPOS=('core') #'extra' 'community' 'testing' 'multilib') +ARCHREPOS=('core' 'testing') #'extra' 'community' 'testing' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos @@ -32,7 +32,7 @@ LOCK_TIMEOUT=300 STAGING="$FTP_BASE/staging" TMPDIR="/tmp" -ARCHARCHES=(i686) # x86_64) +ARCHARCHES=(i686 x86_64) ARCHES=(${ARCHARCHES[@]} mips64el) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" @@ -40,4 +40,4 @@ PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" MAKEPKGCONF="/etc/makepkg.conf" -BLACKLIST_FILE="$FTP_BASE/blacklist.txt" +BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" diff --git a/db-sync b/db-sync index 0bb79b7..1a9e02f 100755 --- a/db-sync +++ b/db-sync @@ -89,7 +89,7 @@ init() { # We delete here for cleanup rsync -vrtlH \ --delete-after \ - --safe-links \ + --delete-excluded \ --delay-updates \ --max-delete=1000 \ --include-from=/tmp/${_repo}-${_arch}.whitelist \ -- cgit v1.2.3-2-g168b From 93255c0baf9beb4f86815a8ba45c7cfa07ccac22 Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 14 Nov 2011 05:18:51 -0800 Subject: Local changes --- config | 21 +++++++++++---------- migrate-repo | 29 +++++++++++++++++++++++++++++ repo-add | 2 +- repo-update | 3 ++- 4 files changed, 43 insertions(+), 12 deletions(-) create mode 100755 migrate-repo diff --git a/config b/config index 0e8dc2e..0e26efb 100644 --- a/config +++ b/config @@ -1,27 +1,27 @@ #!/bin/bash -FTP_BASE="/home/parabolavnx/parabolagnulinux.org/repo" -ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" -SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" +FTP_BASE="/srv/http/repo/public" +ARCH_BASE="/srv/http/repo/public" +SVNREPO="/srv/http/repo/abslibre" # Repos from Arch ARCHREPOS=('core' 'extra' 'community' 'testing' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos -USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm') +USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan') # Community project repos -PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'social') +PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'sugar') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) PKGPOOL='pool/packages' SRCPOOL='sources/packages' CLEANUP_DESTDIR="$FTP_BASE/old/packages" -CLEANUP_DRYRUN=false +CLEANUP_DRYRUN=true # Time in days to keep moved packages CLEANUP_KEEP=30 SOURCE_CLEANUP_DESTDIR="$FTP_BASE/old/sources" -SOURCE_CLEANUP_DRYRUN=false +SOURCE_CLEANUP_DRYRUN=true # Time in days to keep moved sourcepackages SOURCE_CLEANUP_KEEP=30 @@ -31,13 +31,14 @@ LOCK_DELAY=10 LOCK_TIMEOUT=300 STAGING="$FTP_BASE/staging" -TMPDIR="$HOME/tmp" +TMPDIR="/tmp" ARCHARCHES=(i686 x86_64) -ARCHES=(${ARCHARCHES[@]} mips64el) +OURARCHES=(mips64el) +ARCHES=(${ARCHARCHES[@]} ${OURARCHES[@]}) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" -MAKEPKGCONF="$HOME/etc/makepkg.conf" +MAKEPKGCONF="~/.makepkg.conf" BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" diff --git a/migrate-repo b/migrate-repo new file mode 100755 index 0000000..2e44adb --- /dev/null +++ b/migrate-repo @@ -0,0 +1,29 @@ +#!/bin/bash + +source $(dirname $0)/config + +#dryrun="--dry-run" + +# Sync our repos +#for parabola_repo in ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}; do +for parabola_repo in ${USERREPOS[@]} ${PROJREPOS[@]}; do + echo ":: Syncing ${parabola_repo}/ => ${FTP_BASE}/${parabola_repo}/" + rsync -avL ${dryrun} --progress -e ssh parabolavnx@69.163.153.218:repo/${parabola_repo}/ ${FTP_BASE}/${parabola_repo}/ +done + +# Sync our arches +#for parabola_arch in ${OURARCHES[@]}; do +#for arch_repo in ${ARCHREPOS[@]}; do + #echo ":: Syncing ${arch_repo}/os/${parabola_arch}/ => ${FTP_BASE}/${arch_repo}/os/${parabola_arch}/" +# rsync -avL ${dryrun} --progress -e ssh parabolavnx@69.163.153.218:repo/${arch_repo}/os/${parabola_arch}/ ${FTP_BASE}/${arch_repo}/os/${parabola_arch}/ +#done +#done +# +# Sync other dirs last +#for other in screenshots isos; do +for other in other; do + echo ":: Syncing ${other}/ => ${FTP_BASE}/${other}/" + rsync -avL ${dryrun} --progress -e ssh parabolavnx@69.163.153.218:repo/${other}/ ${FTP_BASE}/${other} +done + +exit $? diff --git a/repo-add b/repo-add index c4bf96f..ef9c1e1 100755 --- a/repo-add +++ b/repo-add @@ -20,7 +20,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -LICENSESDIR=/home/parabolavnx/licenses +LICENSESDIR=/srv/http/repo/licenses # gettext initialization export TEXTDOMAIN='pacman' diff --git a/repo-update b/repo-update index 4f31116..bf8d73c 100755 --- a/repo-update +++ b/repo-update @@ -22,7 +22,8 @@ for repo in ${ARCHREPOS[@]}; do # rsync_not_needed for using as an rsync exclude-from #find ${repodir}/${repo}/os/${arch}/ -name "*${PKGEXT}" \ # -fprintf ${rsync_not_needed}-${repo}-${arch} '%f\n' || exit 1 - find ${repodir}/${PKGPOOL}/ -name "*${PKGEXT}" \ + #find ${repodir}/${PKGPOOL}/ -name "*${PKGEXT}" \ + find ${repodir}/${repo}/os/${arch}/ -name "*${PKGEXT}" \ -fprintf ${rsync_not_needed}-${repo}-${arch} '%f\n' || exit 1 # Actual rsync command ${rsync_update_command} \ -- cgit v1.2.3-2-g168b From 9ef417865ef8b102875d42d4cb8ebe277de4ec01 Mon Sep 17 00:00:00 2001 From: Parabola Date: Wed, 23 Nov 2011 07:54:13 -0800 Subject: Making it work --- config | 2 +- db-sync | 63 +++++++++++++++++++++++++++++++++++---------------------------- 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/config b/config index 7122aef..e0d474c 100644 --- a/config +++ b/config @@ -4,7 +4,7 @@ ARCH_BASE="/srv/http/repo/public" SVNREPO="/var/abs" # Repos from Arch -ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib') +ARCHREPOS=('core' 'testing' 'extra' 'community') # 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos diff --git a/db-sync b/db-sync index 1a9e02f..320a71d 100755 --- a/db-sync +++ b/db-sync @@ -18,13 +18,14 @@ # Returns contents of a repo get_repos() { + mkdir -p ${TMPDIR}/${PID}.cache # Exclude everything but db files - rsync -avm --include="*/" \ + rsync -vmrtlH --no-p --include="*/" \ --include="*.db" \ --include="*${DBEXT}" \ --exclude="*" \ --delete-after \ - rsync://${mirror}/${mirrorpath}/ cache/ + rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/${PID}.cache } get_repo_content() { @@ -42,9 +43,9 @@ get_blacklist() { # repo # arch get_repo_file() { - [ ! -r "cache/${1}/os/${2}/${1}${DBEXT}" ] && return 1 +# shopt -s nullglob - echo "cache/${1}/os/${2}/${1}${DBEXT}" + echo "${TMPDIR}/${PID}.cache/${1}/os/${2}/${1}${DBEXT}" } # Process the databases and get the libre packages @@ -61,8 +62,8 @@ init() { get_repos # Traverse all repo-arch pairs - for _arch in ${ARCHARCHES[@]}; do - for _repo in ${ARCHREPOS[@]}; do + for _repo in ${ARCHREPOS[@]}; do + for _arch in ${ARCHARCHES[@]}; do msg "Processing ${_repo}-${_arch}" repo_file=$(get_repo_file ${_repo} ${_arch}) @@ -91,7 +92,6 @@ init() { --delete-after \ --delete-excluded \ --delay-updates \ - --max-delete=1000 \ --include-from=/tmp/${_repo}-${_arch}.whitelist \ --exclude="*" \ rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ @@ -100,17 +100,18 @@ init() { # Add a new whitelist whitelists+=(/tmp/${_repo}-${_arch}.whitelist) + msg "Putting databases back in place" + rsync -vrtlH \ + --delay-updates \ + --safe-links \ + ${TMPDIR}/${PID}.cache/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + # Cleanup unset db done done - msg "Putting databases back in place" - rsync -vrtlH \ - --delay-updates \ - --safe-links \ - cache/ \ - ${FTP_BASE}/ msg "Syncing package pool" # Concatenate all whitelists @@ -121,13 +122,15 @@ init() { # Sync # *Don't delete-after*, this is the job of cleanup scripts. It will remove our # packages too - rsync -vrtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ - ${FTP_BASE}/${PKGPOOL}/ + for PKGPOOL in pool/packages pool/community extra/os/any community/os/any; do + rsync -vrtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ + ${FTP_BASE}/${PKGPOOL}/ + done # Sync sources msg "Syncing source pool" @@ -137,13 +140,15 @@ init() { # Sync # *Don't delete-after*, this is the job of cleanup scripts. It will remove our # packages too - rsync -vrtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ - ${FTP_BASE}/${SRCPOOL}/ + for SRCPOOL in sources/packages sources/community ; do + rsync -vrtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ + ${FTP_BASE}/${SRCPOOL}/ + done # Cleanup @@ -153,7 +158,7 @@ init() { trap_exit() { echo error "$@" - exit 1 + exit 1 } @@ -169,3 +174,5 @@ trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR init + +rm -r ${TMPDIR}/${PID}.cache -- cgit v1.2.3-2-g168b From 51e733fdb525e1c1e9392aa4c3a0b882a6433ad4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 23 Nov 2011 13:19:35 -0300 Subject: Cleanup script + New config vars --- config | 6 ++++++ db-cleanup | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ db-sync | 4 ++-- 3 files changed, 71 insertions(+), 2 deletions(-) create mode 100755 db-cleanup diff --git a/config b/config index e0d474c..47c9c66 100644 --- a/config +++ b/config @@ -15,6 +15,12 @@ PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) PKGPOOL='pool/packages' SRCPOOL='sources/packages' +# Directories where packages are shared between repos +# *relative to FTP_BASE* +PKGPOOLS=('pool/packages' 'pool/community' 'extra/os/any' 'community/os/any') +# Directories where sources are stored +SRCPOOLS=('sources/packages' 'sources/community') + CLEANUP_DESTDIR="$FTP_BASE/old/packages" CLEANUP_DRYRUN=true # Time in days to keep moved packages diff --git a/db-cleanup b/db-cleanup new file mode 100755 index 0000000..0c093c2 --- /dev/null +++ b/db-cleanup @@ -0,0 +1,63 @@ +#!/bin/bash +# Syncs pools against themselves using database contents as filter to cleanup +# them up +# License: GPLv3 + +# Principles +# * Get repos dbs contents +# * Make them a include list +# * Rsync pools against themselves removing excluded files +# * Instant cleanup! + +trap_exit() { + echo + error "$@" + exit 1 +} + +source $(dirname $0)/config +source $(dirname $0)/local_config +source $(dirname $0)/libremessages + +# From makepkg +set -E + +trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT +trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT +trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR + +(( CLEANUP_DRYRUN )) && EXTRAFLAGS+=" --dry-run" + +for _repo in ${ARCHREPOS[@]}; do + for _arch in ${ARCHARCHES[@]}; do + msg "Getting ${_repo}-${_arch} database" + + dbfile="${FTP_BASE}/${_repo}/os/${_arch}/${_repo}${DBEXT}" + + if [ ! -r "${dbfile}" ]; then + warning "Not found" + continue + fi + +# Echo the contents into a filter file + bsdtar tf "${dbfile}" | \ + cut -d'/' -f1 | \ + sort -u | \ + sed "s|$|*|" >> /tmp/$0.$PID.filter + + done +done + +msg "Removing old files:" + +for POOL in ${PKGPOOLS[@]} ${SRCPOOLS[@]}; do + msg2 "${POOL}" + + rsync ${EXTRAFLAGS} -va --delete-excluded \ + --include-from="${dbfile}" \ + --exclude="*" \ + ${FTP_BASE}/${POOL}/ \ + ${FTP_BASE}/${POOL}/ +done + +exit $? diff --git a/db-sync b/db-sync index 320a71d..bcdf41a 100755 --- a/db-sync +++ b/db-sync @@ -122,7 +122,7 @@ init() { # Sync # *Don't delete-after*, this is the job of cleanup scripts. It will remove our # packages too - for PKGPOOL in pool/packages pool/community extra/os/any community/os/any; do + for PKGPOOL in ${PKGPOOLS[@]}; do rsync -vrtlH \ --delay-updates \ --safe-links \ @@ -140,7 +140,7 @@ init() { # Sync # *Don't delete-after*, this is the job of cleanup scripts. It will remove our # packages too - for SRCPOOL in sources/packages sources/community ; do + for SRCPOOL in ${SRCPOOLS[@]}; do rsync -vrtlH \ --delay-updates \ --safe-links \ -- cgit v1.2.3-2-g168b From ee7bba114de818df23196edb7d0d785217b7a332 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 23 Nov 2011 13:24:00 -0300 Subject: DRYRUN check --- db-cleanup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-cleanup b/db-cleanup index 0c093c2..b12e9c1 100755 --- a/db-cleanup +++ b/db-cleanup @@ -26,7 +26,7 @@ trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR -(( CLEANUP_DRYRUN )) && EXTRAFLAGS+=" --dry-run" +${CLEANUP_DRYRUN} && EXTRAFLAGS+=" --dry-run" for _repo in ${ARCHREPOS[@]}; do for _arch in ${ARCHARCHES[@]}; do -- cgit v1.2.3-2-g168b From 4a31562deb872713acae3d17f0f638edd0c4158e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 23 Nov 2011 13:43:04 -0300 Subject: Fixed errors in logic --- db-cleanup | 8 ++++---- db-sync | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/db-cleanup b/db-cleanup index b12e9c1..72790f1 100755 --- a/db-cleanup +++ b/db-cleanup @@ -28,8 +28,8 @@ trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR ${CLEANUP_DRYRUN} && EXTRAFLAGS+=" --dry-run" -for _repo in ${ARCHREPOS[@]}; do - for _arch in ${ARCHARCHES[@]}; do +for _repo in ${PKGREPOS[@]}; do + for _arch in ${ARCHES[@]}; do msg "Getting ${_repo}-${_arch} database" dbfile="${FTP_BASE}/${_repo}/os/${_arch}/${_repo}${DBEXT}" @@ -43,7 +43,7 @@ for _repo in ${ARCHREPOS[@]}; do bsdtar tf "${dbfile}" | \ cut -d'/' -f1 | \ sort -u | \ - sed "s|$|*|" >> /tmp/$0.$PID.filter + sed "s|$|*|" >> /tmp/$0.$$.filter done done @@ -54,7 +54,7 @@ for POOL in ${PKGPOOLS[@]} ${SRCPOOLS[@]}; do msg2 "${POOL}" rsync ${EXTRAFLAGS} -va --delete-excluded \ - --include-from="${dbfile}" \ + --include-from="/tmp/$0.$$.filter" \ --exclude="*" \ ${FTP_BASE}/${POOL}/ \ ${FTP_BASE}/${POOL}/ diff --git a/db-sync b/db-sync index bcdf41a..110dba1 100755 --- a/db-sync +++ b/db-sync @@ -18,14 +18,14 @@ # Returns contents of a repo get_repos() { - mkdir -p ${TMPDIR}/${PID}.cache + mkdir -p ${TMPDIR}/$0.$$.cache # Exclude everything but db files rsync -vmrtlH --no-p --include="*/" \ --include="*.db" \ --include="*${DBEXT}" \ --exclude="*" \ --delete-after \ - rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/${PID}.cache + rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/$0.$$.cache } get_repo_content() { @@ -45,7 +45,7 @@ get_blacklist() { get_repo_file() { # shopt -s nullglob - echo "${TMPDIR}/${PID}.cache/${1}/os/${2}/${1}${DBEXT}" + echo "${TMPDIR}/$0.$$.cache/${1}/os/${2}/${1}${DBEXT}" } # Process the databases and get the libre packages @@ -104,7 +104,7 @@ init() { rsync -vrtlH \ --delay-updates \ --safe-links \ - ${TMPDIR}/${PID}.cache/${_repo}/os/${_arch}/ \ + ${TMPDIR}/$0.$$.cache/${_repo}/os/${_arch}/ \ ${FTP_BASE}/${_repo}/os/${_arch}/ # Cleanup @@ -175,4 +175,4 @@ trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR init -rm -r ${TMPDIR}/${PID}.cache +rm -r ${TMPDIR}/$0.$$.cache -- cgit v1.2.3-2-g168b From 323b2e1d83af45b7e4fe90c4822d3366cc48e588 Mon Sep 17 00:00:00 2001 From: Parabola Date: Wed, 7 Dec 2011 06:04:08 -0800 Subject: Added [multilib] --- config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config b/config index 47c9c66..92b7bfa 100644 --- a/config +++ b/config @@ -4,11 +4,11 @@ ARCH_BASE="/srv/http/repo/public" SVNREPO="/var/abs" # Repos from Arch -ARCHREPOS=('core' 'testing' 'extra' 'community') # 'multilib') +ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos -USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan') +USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu') # Community project repos PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'sugar') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) @@ -22,7 +22,7 @@ PKGPOOLS=('pool/packages' 'pool/community' 'extra/os/any' 'community/os/any') SRCPOOLS=('sources/packages' 'sources/community') CLEANUP_DESTDIR="$FTP_BASE/old/packages" -CLEANUP_DRYRUN=true +CLEANUP_DRYRUN=false # Time in days to keep moved packages CLEANUP_KEEP=30 -- cgit v1.2.3-2-g168b From d2d3d2785d622b9bd8539deca4d79ccc4a3d52c4 Mon Sep 17 00:00:00 2001 From: Parabola Date: Wed, 7 Dec 2011 06:04:39 -0800 Subject: Skip invalid repo-arch pairs (useful for [multilib]) --- db-sync | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/db-sync b/db-sync index 110dba1..d86d348 100755 --- a/db-sync +++ b/db-sync @@ -68,6 +68,11 @@ init() { repo_file=$(get_repo_file ${_repo} ${_arch}) + if [ ! -f "${repo_file}" ]; then + warning "${repo_file} doesn't exist, skipping this repo-arch" + continue + fi + # Remove blacklisted packages and count them # TODO capture all removed packages for printing on debug mode msg2 "Removing blacklisted packages: $( -- cgit v1.2.3-2-g168b From 2932fb1b926827c2e12eccd7eb10c3116b6835a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 7 Dec 2011 11:55:06 -0300 Subject: Any-to-ours recycles Arch packages ("any" architecture) to Parabola specific architectures ("mips64el") --- any-to-ours | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100755 any-to-ours diff --git a/any-to-ours b/any-to-ours new file mode 100755 index 0000000..2fa8323 --- /dev/null +++ b/any-to-ours @@ -0,0 +1,67 @@ +#!/bin/bash +# Releases 'any' packages from Arch arches to ours + +trap_exit() { + echo + error "$@" + exit 1 +} + +source $(dirname $0)/config +source $(dirname $0)/local_config +source $(dirname $0)/libremessages + +# From makepkg +set -E + +trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT +trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT +trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR + +# The architecture to compare with +BASEARCH='x86_64' + +# Traverse all Arch repos +for _repo in ${ARCHREPOS[@]}; do + msg "Processing ${_repo}..." + +# Find 'any' packages +# This is hardcoded but it could release other arches... + PKGS=($(find "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \ + -iname '*-any.pkg.tar.?z' \ + -printf "%f ")) + + if [ ${#PKGS[@]} -eq 0 ]; then + msg2 "No 'any' packages here" + continue + fi + + for _arch in ${OURARCHES[@]}; do + msg2 "Syncing ${_arch}..." + +# Sync 'any' only and extract the synced packages + SYNCED=($( + rsync -av \ + --include='*-any.pkg.tar.?z' \ + --exclude='*' \ + ${FTP_BASE}/${_repo}/os/${BASEARCH}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ 2>&1 | \ + grep '-any.pkg.tar' | \ + cut -d ' ' -f 1 )) + + msg2 "Synced ${#SYNCED[@]} packages" + + msg2 "Adding to db..." + + pushd ${FTP_BASE}/${_repo}/os/${_arch}/ >/dev/null + +# Add the packages to the db + $(dirname $0)/repo-add ${_repo}${DBEXT} \ + ${SYNCED[@]} + + popd >/dev/null + +# Avoid mixups + unset SYNCED PKGS + done +done -- cgit v1.2.3-2-g168b From 6fedfa4233c8e237d75ddec6ac7122e051f509a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 7 Dec 2011 12:10:26 -0300 Subject: Removed extra - --- any-to-ours | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/any-to-ours b/any-to-ours index 2fa8323..8c37622 100755 --- a/any-to-ours +++ b/any-to-ours @@ -46,7 +46,7 @@ for _repo in ${ARCHREPOS[@]}; do --exclude='*' \ ${FTP_BASE}/${_repo}/os/${BASEARCH}/ \ ${FTP_BASE}/${_repo}/os/${_arch}/ 2>&1 | \ - grep '-any.pkg.tar' | \ + grep 'any.pkg.tar' | \ cut -d ' ' -f 1 )) msg2 "Synced ${#SYNCED[@]} packages" -- cgit v1.2.3-2-g168b From 56b914d69d96cc410d9bf385c18bcb128af83a58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 7 Dec 2011 12:11:36 -0300 Subject: Added synced check --- any-to-ours | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/any-to-ours b/any-to-ours index 8c37622..1bd7e8a 100755 --- a/any-to-ours +++ b/any-to-ours @@ -49,6 +49,11 @@ for _repo in ${ARCHREPOS[@]}; do grep 'any.pkg.tar' | \ cut -d ' ' -f 1 )) + if [ ${#SYNCED[@]} -eq 0 ]; then + msg2 "Already synced (or error happened)" + continue + fi + msg2 "Synced ${#SYNCED[@]} packages" msg2 "Adding to db..." -- cgit v1.2.3-2-g168b From a749983e8f2becf7c5427b0e5c449d457afbe385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 7 Dec 2011 12:17:58 -0300 Subject: Be verbose --- any-to-ours | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/any-to-ours b/any-to-ours index 1bd7e8a..15a20be 100755 --- a/any-to-ours +++ b/any-to-ours @@ -54,7 +54,7 @@ for _repo in ${ARCHREPOS[@]}; do continue fi - msg2 "Synced ${#SYNCED[@]} packages" + msg2 "Synced ${#SYNCED[@]} packages: ${SYNCED[@]}" msg2 "Adding to db..." -- cgit v1.2.3-2-g168b From 59105fccf9f5866b451cdaf90052aa8c9346e5f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sun, 8 Jan 2012 15:17:46 -0300 Subject: remove locks --- cron-jobs/ftpdir-cleanup | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index bb1661a..83e6e17 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -22,13 +22,6 @@ clean_pkg() { fi } -script_lock - -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do - repo_lock ${repo} ${arch} || exit 1 - done -done ${CLEANUP_DRYRUN} && warning 'dry run mode is active' -- cgit v1.2.3-2-g168b From 3b719e6ffcc9a4e30ba9ecb00de7be0e1e11bc46 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 8 Jan 2012 18:26:20 +0000 Subject: [gnu] repo --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 92b7bfa..949d171 100644 --- a/config +++ b/config @@ -10,7 +10,7 @@ OURREPOS=('libre' 'libre-testing') # User repos USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu') # Community project repos -PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'sugar') +PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) PKGPOOL='pool/packages' SRCPOOL='sources/packages' -- cgit v1.2.3-2-g168b From 6e90f39ff729714516779f96875dc9b46854b094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sun, 8 Jan 2012 15:27:12 -0300 Subject: Get_repos scripts updates parabolaweb package database --- get_repos | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100755 get_repos diff --git a/get_repos b/get_repos new file mode 100755 index 0000000..824ab15 --- /dev/null +++ b/get_repos @@ -0,0 +1,33 @@ +#!/bin/bash +# TODO needs refactoring + +PDIR="/srv/http/web" +DIR="/srv/http/web/repos" + +for arch in i686 x86_64 mips64el; do + for repo in community extra core testing libre libre-testing social; do + [[ ! -d $DIR/$arch ]] && mkdir -p $DIR/$arch + + cd $DIR/$arch + + rm -f *.db.tar.gz + + wget http://repo.parabolagnulinux.org/$repo/os/$arch/$repo.db.tar.gz && \ + $PDIR/manage.py reporead $arch $DIR/$arch/$repo.db.tar.gz + done +done + +for repo in connos connos-extra; do + arch=i586 + [[ ! -d $DIR/$arch ]] && mkdir -p $DIR/$arch + + cd $DIR/$arch + + rm -f *.db.tar.gz + + wget http://www.connochaetos.org/os/$arch/$repo/$repo.db.tar.gz && \ + $PDIR/manage.py reporead $arch $DIR/$arch/$repo.db.tar.gz +done + + +exit 0 -- cgit v1.2.3-2-g168b From a3e0b61d4d14f01442b98deaa18f66ceb3c4b3b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sun, 8 Jan 2012 15:27:34 -0300 Subject: Use standar naming --- get-repos | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100755 get-repos diff --git a/get-repos b/get-repos new file mode 100755 index 0000000..824ab15 --- /dev/null +++ b/get-repos @@ -0,0 +1,33 @@ +#!/bin/bash +# TODO needs refactoring + +PDIR="/srv/http/web" +DIR="/srv/http/web/repos" + +for arch in i686 x86_64 mips64el; do + for repo in community extra core testing libre libre-testing social; do + [[ ! -d $DIR/$arch ]] && mkdir -p $DIR/$arch + + cd $DIR/$arch + + rm -f *.db.tar.gz + + wget http://repo.parabolagnulinux.org/$repo/os/$arch/$repo.db.tar.gz && \ + $PDIR/manage.py reporead $arch $DIR/$arch/$repo.db.tar.gz + done +done + +for repo in connos connos-extra; do + arch=i586 + [[ ! -d $DIR/$arch ]] && mkdir -p $DIR/$arch + + cd $DIR/$arch + + rm -f *.db.tar.gz + + wget http://www.connochaetos.org/os/$arch/$repo/$repo.db.tar.gz && \ + $PDIR/manage.py reporead $arch $DIR/$arch/$repo.db.tar.gz +done + + +exit 0 -- cgit v1.2.3-2-g168b From 47b0e5207b933e5563324c92c8a1841322cde6e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Mon, 9 Jan 2012 15:34:13 -0300 Subject: Use config ARCHES and REPOS to get-repos --- config | 5 +++++ get-repos | 71 ++++++++++++++++++++++++++++++++++++++++++++++----------------- get_repos | 33 ----------------------------- 3 files changed, 57 insertions(+), 52 deletions(-) delete mode 100755 get_repos diff --git a/config b/config index 949d171..1304e01 100644 --- a/config +++ b/config @@ -11,6 +11,8 @@ OURREPOS=('libre' 'libre-testing') USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu') # Community project repos PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu') +# Remote repos +RMTREPOS=('connos' 'connos-extra') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) PKGPOOL='pool/packages' SRCPOOL='sources/packages' @@ -48,3 +50,6 @@ SRCEXT=".src.tar.gz" MAKEPKGCONF="~/.makepkg.conf" BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" + +# parabolaweb root +WEB_DIR=/srv/http/web diff --git a/get-repos b/get-repos index 824ab15..9ff701c 100755 --- a/get-repos +++ b/get-repos @@ -1,33 +1,66 @@ #!/bin/bash -# TODO needs refactoring +# Gets repo databases and updates parabolaweb +# Note: It works remotely because our parabolaweb server and repo server are +# two different hosts -PDIR="/srv/http/web" -DIR="/srv/http/web/repos" +trap_exit() { + echo + error "$@" + exit 1 +} -for arch in i686 x86_64 mips64el; do - for repo in community extra core testing libre libre-testing social; do - [[ ! -d $DIR/$arch ]] && mkdir -p $DIR/$arch +source $(dirname $0)/config +source $(dirname $0)/local_config +source $(dirname $0)/libremessages - cd $DIR/$arch +# From makepkg +set -E - rm -f *.db.tar.gz +trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT +trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT +trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR - wget http://repo.parabolagnulinux.org/$repo/os/$arch/$repo.db.tar.gz && \ - $PDIR/manage.py reporead $arch $DIR/$arch/$repo.db.tar.gz +TMPDIR="$(mktemp -d /tmp/$0.XXXX)" +DBLIST=() + +# Repos +for _repo in ${PKGREPOS[@]}; do + for _arch in ${ARCHES[@]}; do + DBLIST+=("http://repo.parabolagnulinux.org/${_repo}/os/${_arch}/${_repo}${DBEXT}") done done -for repo in connos connos-extra; do - arch=i586 - [[ ! -d $DIR/$arch ]] && mkdir -p $DIR/$arch +# Remote repos +# TODO don't hardcode the remote architecture +# TODO don't hardcode the remote url +# MAYBE run scripts on get-repos.d/ which should return db urls +for _repo in ${RMTREPOS}; do + _arch=i586 + DBLIST+=("http://www.connochaetos.org/os/${_arch}/${_repo}/${_repo}${DBEXT}") +done - cd $DIR/$arch +# Get them all +msg "Retrieving ${#DBLIST[@]} databases" +wget --directory-prefix=${TMPDIR} \ + --no-verbose \ + --force-directories \ + --no-host-directories \ + ${DBLIST[@]} || true +# Always return true, some databases are expect to be missing - rm -f *.db.tar.gz +# Create the arches regexp arch1|arch2 +arch_re="$(echo "(${ARCHES[@]} i586)" | tr ' ' '|')" - wget http://www.connochaetos.org/os/$arch/$repo/$repo.db.tar.gz && \ - $PDIR/manage.py reporead $arch $DIR/$arch/$repo.db.tar.gz -done +msg "Adding to parabolaweb" +find "${TMPDIR}" -iname "*${DBEXT}" | while read _db; do + _arch=$(echo "${_db}" | egrep -o "${arch_re}") + if [ -z "${_arch}" ]; then + error "Can't find database architecture: ${_db}" + continue + fi + + "${WEB_DIR}"/manage.py reporead "${_arch}" "${_db}" +done -exit 0 +exit $? diff --git a/get_repos b/get_repos deleted file mode 100755 index 824ab15..0000000 --- a/get_repos +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# TODO needs refactoring - -PDIR="/srv/http/web" -DIR="/srv/http/web/repos" - -for arch in i686 x86_64 mips64el; do - for repo in community extra core testing libre libre-testing social; do - [[ ! -d $DIR/$arch ]] && mkdir -p $DIR/$arch - - cd $DIR/$arch - - rm -f *.db.tar.gz - - wget http://repo.parabolagnulinux.org/$repo/os/$arch/$repo.db.tar.gz && \ - $PDIR/manage.py reporead $arch $DIR/$arch/$repo.db.tar.gz - done -done - -for repo in connos connos-extra; do - arch=i586 - [[ ! -d $DIR/$arch ]] && mkdir -p $DIR/$arch - - cd $DIR/$arch - - rm -f *.db.tar.gz - - wget http://www.connochaetos.org/os/$arch/$repo/$repo.db.tar.gz && \ - $PDIR/manage.py reporead $arch $DIR/$arch/$repo.db.tar.gz -done - - -exit 0 -- cgit v1.2.3-2-g168b From 88773a8de82e8064fcf1e9fb6b73d64c717b7cd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Mon, 23 Jan 2012 12:06:56 -0300 Subject: Cross-toolchains repo --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 1304e01..4de9d00 100644 --- a/config +++ b/config @@ -10,7 +10,7 @@ OURREPOS=('libre' 'libre-testing') # User repos USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu') # Community project repos -PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu') +PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross') # Remote repos RMTREPOS=('connos' 'connos-extra') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) -- cgit v1.2.3-2-g168b From 864dae08b0431febd00349e60c73e392f8be8ab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Mon, 30 Jan 2012 19:59:18 -0300 Subject: ABSLibre creation script --- abslibre | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100755 abslibre diff --git a/abslibre b/abslibre new file mode 100755 index 0000000..9485986 --- /dev/null +++ b/abslibre @@ -0,0 +1,93 @@ +#!/bin/bash + +ABSLIBRE=/var/abslibre +ABSGIT=/srv/git/repositories/abslibre.git +# Remote +# ABSGIT=http://projects.parabolagnulinux.org/abslibre.git +BLACKLIST='http://repo.parabolagnulinux.org/docs/blacklist.txt' +SYNCARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g' +BLFILE=/tmp/blacklist.txt + +. /etc/abs.conf + +# Steps +# * Sync abs +# * Download blacklist.txt +# * Sync abslibre from abs excluding from blacklist +# * Create repo.abs.tar.gz tarballs + +function sync_abs() { + for ARCH in any i686 x86_64; do + rsync ${SYNCARGS} ${SYNCSERVER}::abs/${ARCH}/ ${ABSROOT}/${ARCH} || return $? + done +} + +function get_blacklist() { + printf ":: Updating blacklist...\t" + wget -q -O - "${BLACKLIST}" | cut -d':' -f1 | sort -u | \ + sed "s/^/**\//" > ${BLFILE} || { + printf "[FAILED]\n" + return 1 + } + + printf "[OK]\n" +} + +function sync_abs_libre() { + +# Clone ABSLibre git repo + if [ -d /tmp/abslibre/.git ]; then + pushd /tmp/abslibre >/dev/null 2>&1 + git pull + popd >/dev/null 2>&1 + else + git clone /srv/git/repositories/abslibre.git /tmp/abslibre + fi + +# Sync from ABS and then sync from ABSLibre + printf ":: Syncing ABSLibre...\t" + (rsync ${SYNCARGS} --delete-excluded \ + --exclude-from=${BLFILE} \ + ${ABSROOT} \ + ${ABSLIBRE} \ + && + for ARCH in i686 x86_64; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --exclude=.git/ /tmp/abslibre/ ${ABSLIBRE}/${ARCH}/; done) || { + printf "[FAILED]\n" + return 1 + } + + printf "[OK]\n" +} + +sync_pre_mips64el() { + pushd /home/parabola/abslibre-pre-mips64el >/dev/null + + rsync ${SYNCARGS} --exclude=.git* ${ABSLIBRE}/x86_64/ /home/parabola/abslibre-pre-mips64el/ && git add * && git commit -m "$(date)" +} + +# Create .abs.tar.gz tarballs +create_tarballs() { + for repo in ${ABSLIBRE}/{i686,x86_64}/*; do + baserepo=$(basename $repo) + arch=$(basename $(dirname $repo)) + +# Remove the old one + mkdir -p /srv/http/web/media/abs/$baserepo/os/$arch/ + rm /srv/http/web/media/abs/$baserepo/os/$arch/$baserepo.abs.tar.gz +# Create a new one joining arch and any +# Remove the first part of the path (it could be $repo but any isn't hit) + bsdtar -czvf /srv/http/web/media/abs/$baserepo/os/$arch/$baserepo.abs.tar.gz \ + -s ":${ABSLIBRE}/[a-z0-9_]\+/[a-z]\+::" \ + $repo/* ${ABSLIBRE}/any/${baserepo}/* + + done +} + +sync_abs || exit 1 +get_blacklist || exit 1 +sync_abs_libre || exit 1 +# This is being done at repo server now +sync_pre_mips64el || exit 1 +create_tarballs || exit 1 + +exit 0 -- cgit v1.2.3-2-g168b From 4c6b6514e5eb78742369bea3e79fa2c749c1741a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Mas=C5=82owski?= Date: Wed, 1 Feb 2012 16:37:10 +0100 Subject: check-package-libraries.py, aiming to help issue224. --- COPYING | 674 +++++++++++++++++++++++++++++++++++++++++++++ check-package-libraries.py | 193 +++++++++++++ 2 files changed, 867 insertions(+) create mode 100644 COPYING create mode 100755 check-package-libraries.py diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..94a9ed0 --- /dev/null +++ b/COPYING @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/check-package-libraries.py b/check-package-libraries.py new file mode 100755 index 0000000..bc2349b --- /dev/null +++ b/check-package-libraries.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +# Copyright (C) 2012 Michał Masłowski +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +Check which libraries are provided or required by a package, store +this in a database, update and list broken packages. + +Dependencies: + +- Python 3.2 or later with SQLite 3 support + +- ``bsdtar`` + +- ``readelf`` +""" + + +import os.path +import re +import sqlite3 +import subprocess +import tempfile + + +#: Regexp matching an interesting dynamic entry. +_DYNAMIC = re.compile(r"^\s*[0-9a-fx]+" + "\s*\((NEEDED|SONAME)\)[^:]*:\s*\[(.+)\]$") + + +def make_db(path): + """Make a new, empty, library database at *path*.""" + con = sqlite3.connect(path) + con.executescript(""" +create table provided( + library varchar not null, + package varchar not null +); +create table used( + library varchar not null, + package varchar not null +); +""") + con.close() + + +def begin(database): + """Connect to *database* and start a transaction.""" + con = sqlite3.connect(database) + con.execute("begin exclusive") + return con + + +def add_provided(con, package, libraries): + """Write that *package* provides *libraries*.""" + for library in libraries: + con.execute("insert into provided (package, library) values (?,?)", + (package, library)) + + +def add_used(con, package, libraries): + """Write that *package* uses *libraries*.""" + for library in libraries: + con.execute("insert into used (package, library) values (?,?)", + (package, library)) + + +def remove_package(con, package): + """Remove all entries for a package.""" + con.execute("delete from provided where package=?", (package,)) + con.execute("delete from used where package=?", (package,)) + + +def add_package(con, package): + """Add entries from a named *package*.""" + # Extract to a temporary directory. This could be done more + # efficiently, since there is no need to store more than one file + # at once. + with tempfile.TemporaryDirectory() as temp: + tar = subprocess.Popen(("bsdtar", "xf", package, "-C", temp)) + tar.communicate() + with open(os.path.join(temp, ".PKGINFO")) as pkginfo: + for line in pkginfo: + if line.startswith("pkgname ="): + pkgname = line[len("pkgname ="):].strip() + break + # Don't list previously removed libraries. + remove_package(con, pkgname) + provided = set() + used = set() + # Search for ELFs. + for dirname, dirnames, filenames in os.walk(temp): + assert dirnames is not None # unused, avoid pylint warning + for file_name in filenames: + path = os.path.join(dirname, file_name) + with open(path, "rb") as file_object: + if file_object.read(4) != b"\177ELF": + continue + readelf = subprocess.Popen(("readelf", "-d", path), + stdout=subprocess.PIPE) + for line in readelf.communicate()[0].split(b"\n"): + match = _DYNAMIC.match(line.decode("ascii")) + if match: + if match.group(1) == "SONAME": + provided.add(match.group(2)) + elif match.group(1) == "NEEDED": + used.add(match.group(2)) + else: + raise AssertionError("unknown entry type " + + match.group(1)) + add_provided(con, pkgname, provided) + add_used(con, pkgname, used) + + +def init(arguments): + """Initialize.""" + make_db(arguments.database) + + +def add(arguments): + """Add packages.""" + con = begin(arguments.database) + for package in arguments.packages: + add_package(con, package) + con.commit() + con.close() + + +def remove(arguments): + """Remove packages.""" + con = begin(arguments.database) + for package in arguments.packages: + remove_package(con, package) + con.commit() + con.close() + + +def check(arguments): + """List broken packages.""" + con = begin(arguments.database) + available = set(row[0] for row + in con.execute("select library from provided")) + for package, library in con.execute("select package, library from used"): + if library not in available: + print(package, "needs", library) + con.close() + + +def main(): + """Get arguments and run the command.""" + from argparse import ArgumentParser + parser = ArgumentParser(prog="check-package-libraries.py", + description="Check packages for " + "provided/needed libraries") + parser.add_argument("-d", "--database", type=str, + help="Database file to use", + default="package-libraries.sqlite") + subparsers = parser.add_subparsers() + subparser = subparsers.add_parser(name="init", + help="initialize the database") + subparser.set_defaults(command=init) + subparser = subparsers.add_parser(name="add", + help="add packages to database") + subparser.add_argument("packages", nargs="+", type=str, + help="package files to add") + subparser.set_defaults(command=add) + subparser = subparsers.add_parser(name="remove", + help="remove packages from database") + subparser.add_argument("packages", nargs="+", type=str, + help="package names to remove") + subparser.set_defaults(command=remove) + subparser = subparsers.add_parser(name="check", + help="list broken packages") + subparser.set_defaults(command=check) + arguments = parser.parse_args() + arguments.command(arguments) + + +if __name__ == "__main__": + main() -- cgit v1.2.3-2-g168b From 834327b7defe8273f7978328ad1b822960cd5ba0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Tue, 14 Feb 2012 12:00:50 -0300 Subject: Ignore rsync temp files --- db-functions | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/db-functions b/db-functions index 4dddcb5..4c247a7 100644 --- a/db-functions +++ b/db-functions @@ -293,6 +293,9 @@ getpkgfile() { } getpkgfiles() { +# Ignore anything that doesn't glob to PKGEXT + shopt -s nullglob + local f if [ ! -z "$(echo ${@%\.*} | sed "s/ /\n/g" | sort | uniq -D)" ]; then error 'Duplicate packages found!' @@ -310,6 +313,8 @@ getpkgfiles() { done echo ${@} + + shopt -u nullglob } check_pkgfile() { -- cgit v1.2.3-2-g168b From c90a9245b331f43f369b4ef59e67cb2b16c72988 Mon Sep 17 00:00:00 2001 From: Parabola Date: Wed, 15 Feb 2012 20:30:26 +0000 Subject: Use basename --- get-repos | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/get-repos b/get-repos index 9ff701c..8a96125 100755 --- a/get-repos +++ b/get-repos @@ -20,7 +20,7 @@ trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR -TMPDIR="$(mktemp -d /tmp/$0.XXXX)" +TMPDIR="$(mktemp -d /tmp/$(basename $0).XXXX)" DBLIST=() # Repos -- cgit v1.2.3-2-g168b From 75ad12313337cfb57fed8633b667e1fa8e3d48db Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 27 Feb 2012 19:34:41 +0000 Subject: Added user repos --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 4de9d00..af31f37 100644 --- a/config +++ b/config @@ -8,7 +8,7 @@ ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos -USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu') +USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien') # Community project repos PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross') # Remote repos -- cgit v1.2.3-2-g168b From 46224d22f8f1f59144b1feab2abd7aca18901e9a Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 27 Feb 2012 19:46:45 +0000 Subject: [extra] doesn't have the any dir anymore --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index af31f37..b1a7308 100644 --- a/config +++ b/config @@ -19,7 +19,7 @@ SRCPOOL='sources/packages' # Directories where packages are shared between repos # *relative to FTP_BASE* -PKGPOOLS=('pool/packages' 'pool/community' 'extra/os/any' 'community/os/any') +PKGPOOLS=('pool/packages' 'pool/community' 'community/os/any') # Directories where sources are stored SRCPOOLS=('sources/packages' 'sources/community') -- cgit v1.2.3-2-g168b From 623ccccff54b5497708debd0a6271d8143d03257 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Thu, 22 Mar 2012 12:02:45 -0300 Subject: community/os/any doesn't exist anymore --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index b1a7308..cb552e4 100644 --- a/config +++ b/config @@ -19,7 +19,7 @@ SRCPOOL='sources/packages' # Directories where packages are shared between repos # *relative to FTP_BASE* -PKGPOOLS=('pool/packages' 'pool/community' 'community/os/any') +PKGPOOLS=('pool/packages' 'pool/community') # Directories where sources are stored SRCPOOLS=('sources/packages' 'sources/community') -- cgit v1.2.3-2-g168b From a9680de1b0b37d8ceba6ab90864681d0bf4d7fb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 11 Apr 2012 18:16:02 -0300 Subject: Git-PBS forks packages from Arch svntogit/packages.git It creates a single package git repo and tracks upstream changes. It's intended to fork upstream PKGBUILDs into their -libre versions. **STILL UNDER TEST** --- git-pbs | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100755 git-pbs diff --git a/git-pbs b/git-pbs new file mode 100755 index 0000000..b815863 --- /dev/null +++ b/git-pbs @@ -0,0 +1,44 @@ +#!/bin/bash + +_pkg=$1 + +mkdir -p $_pkg +pushd $_pkg + + +if [ ! -d .git ]; then +# Start a git repo for the package +# Add the remote origin +# Pull the package branch onto an unmodified branch + git init + git remote add arch git://projects.archlinux.org/svntogit/packages.git + +# Export the repository + touch .git/git-daemon-export-ok + +# Pass the -b flag to checkout to create the branches + extra="-b" +fi + +git checkout ${extra} upstream +git pull arch packages/$_pkg + +# Move PKGBUILD and files to the basedir +# Remove everything else from the repo +git checkout ${extra} master + +# This produces a lot of merging conflicts +git merge upstream + +# This apparently solves them +git mv trunk/* . +git rm -rf repos + +# Remove the actual files +rm -rf trunk repos + +# Commit everything +git commit -a -m "Converted to PBS" + +# Return to the repo +popd >/dev/null -- cgit v1.2.3-2-g168b From 65a4d2f541a4292efe422f829d5eac41c548efd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Wed, 18 Apr 2012 18:42:20 -0300 Subject: adding artistic repo to community project --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index cb552e4..b3a4658 100644 --- a/config +++ b/config @@ -10,7 +10,7 @@ OURREPOS=('libre' 'libre-testing') # User repos USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien') # Community project repos -PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross') +PROJREPOS=('social' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross') # Remote repos RMTREPOS=('connos' 'connos-extra') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) -- cgit v1.2.3-2-g168b From 6942fc113939c487a4292589225d0cc60a6a4222 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Fri, 20 Apr 2012 02:07:01 -0300 Subject: adding jorginho repo --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index b3a4658..e6f52e1 100644 --- a/config +++ b/config @@ -8,7 +8,7 @@ ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos -USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien') +USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho') # Community project repos PROJREPOS=('social' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross') # Remote repos -- cgit v1.2.3-2-g168b From 6aa0446283a8ee995fae1e5da196a8027032d68d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Thu, 26 Apr 2012 15:41:01 -0300 Subject: [gis] repo for geospatial software --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index e6f52e1..4909b6d 100644 --- a/config +++ b/config @@ -10,7 +10,7 @@ OURREPOS=('libre' 'libre-testing') # User repos USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho') # Community project repos -PROJREPOS=('social' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross') +PROJREPOS=('social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross') # Remote repos RMTREPOS=('connos' 'connos-extra') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) -- cgit v1.2.3-2-g168b From bd8ba6724d09ca83a533f5641fc84ad3ac492e58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Silva?= Date: Mon, 21 May 2012 16:11:56 -0300 Subject: adding coadde repo --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 4909b6d..c582bea 100644 --- a/config +++ b/config @@ -8,7 +8,7 @@ ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos -USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho') +USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde') # Community project repos PROJREPOS=('social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross') # Remote repos -- cgit v1.2.3-2-g168b From f43b4881b168b951f39dd00f075c8d47ef17df4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Tue, 22 May 2012 11:27:20 -0300 Subject: adding pcr (Parabola Community Repo) --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index c582bea..20ad307 100644 --- a/config +++ b/config @@ -10,7 +10,7 @@ OURREPOS=('libre' 'libre-testing') # User repos USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde') # Community project repos -PROJREPOS=('social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross') +PROJREPOS=('pcr' 'social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross') # Remote repos RMTREPOS=('connos' 'connos-extra') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) -- cgit v1.2.3-2-g168b From 4d3cf060c4a7d2b671d9cdf0ab0f6c1d89a8204b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Mon, 28 May 2012 14:03:30 -0300 Subject: Remove TMPDIR --- get-repos | 2 ++ 1 file changed, 2 insertions(+) diff --git a/get-repos b/get-repos index 8a96125..86b371c 100755 --- a/get-repos +++ b/get-repos @@ -63,4 +63,6 @@ find "${TMPDIR}" -iname "*${DBEXT}" | while read _db; do "${WEB_DIR}"/manage.py reporead "${_arch}" "${_db}" done +rm -r ${TMPDIR} + exit $? -- cgit v1.2.3-2-g168b From 7592118ed1c8952d05f1239e3b84d85d39ea2982 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Tue, 26 Jun 2012 16:06:53 -0300 Subject: Re-enabled check on other repos; skip already released packages --- db-update | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db-update b/db-update index 86eaa2e..1fe5256 100755 --- a/db-update +++ b/db-update @@ -35,9 +35,9 @@ for repo in ${repos[@]}; do if ! check_pkgfile "${pkg}"; then die "Package ${repo}/$(basename ${pkg}) is not consistent with its meta data" fi - #if ! check_pkgrepos "${pkg}"; then - # die "Package ${repo}/$(basename ${pkg}) already exists in another repository" - #fi + if ! check_pkgrepos "${pkg}"; then + die "Package ${repo}/$(basename ${pkg}) already exists in another repository" + fi done # This is fucking obnoxious # if ! check_splitpkgs ${repo} ${pkgs[@]}; then @@ -61,7 +61,7 @@ for repo in ${repos[@]}; do if [ -f "${pkg}" ]; then mv "${pkg}" "$FTP_BASE/${PKGPOOL}" fi - ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" + ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" 2>/dev/null || { error "Package already in repo, skipping" ; continue } # also move signatures if [ -f "${pkg}.sig" ]; then mv "${pkg}.sig" "$FTP_BASE/${PKGPOOL}" -- cgit v1.2.3-2-g168b From 0e43f902b1056bfbed57cba6bb299a75fcdb19a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Tue, 21 Aug 2012 16:37:28 -0300 Subject: Sync signatures too --- any-to-ours | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/any-to-ours b/any-to-ours index 15a20be..3a58946 100755 --- a/any-to-ours +++ b/any-to-ours @@ -43,10 +43,11 @@ for _repo in ${ARCHREPOS[@]}; do SYNCED=($( rsync -av \ --include='*-any.pkg.tar.?z' \ + --include='*-any.pkg.tar.?z.sig' \ --exclude='*' \ ${FTP_BASE}/${_repo}/os/${BASEARCH}/ \ ${FTP_BASE}/${_repo}/os/${_arch}/ 2>&1 | \ - grep 'any.pkg.tar' | \ + grep 'any\.pkg\.tar\..z$' | \ cut -d ' ' -f 1 )) if [ ${#SYNCED[@]} -eq 0 ]; then -- cgit v1.2.3-2-g168b From 9a535045c075d90a962d2817651140feaeb59952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Tue, 21 Aug 2012 17:15:33 -0300 Subject: Fix --- db-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-update b/db-update index 1fe5256..7a8d6d1 100755 --- a/db-update +++ b/db-update @@ -61,7 +61,7 @@ for repo in ${repos[@]}; do if [ -f "${pkg}" ]; then mv "${pkg}" "$FTP_BASE/${PKGPOOL}" fi - ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" 2>/dev/null || { error "Package already in repo, skipping" ; continue } + ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" 2>/dev/null || { error "Package already in repo, removing and skipping" ; continue 2; } # also move signatures if [ -f "${pkg}.sig" ]; then mv "${pkg}.sig" "$FTP_BASE/${PKGPOOL}" -- cgit v1.2.3-2-g168b From d5aa8796a3b3e7314ab2ae0ec566e38a8de8bc28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Tue, 21 Aug 2012 17:16:24 -0300 Subject: This is clearly not the way --- db-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-update b/db-update index 7a8d6d1..875b87f 100755 --- a/db-update +++ b/db-update @@ -61,7 +61,7 @@ for repo in ${repos[@]}; do if [ -f "${pkg}" ]; then mv "${pkg}" "$FTP_BASE/${PKGPOOL}" fi - ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" 2>/dev/null || { error "Package already in repo, removing and skipping" ; continue 2; } + ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" # also move signatures if [ -f "${pkg}.sig" ]; then mv "${pkg}.sig" "$FTP_BASE/${PKGPOOL}" -- cgit v1.2.3-2-g168b From 5f5b5b10e700af5d7e6be352cde1e1e3380e1d3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Mas=C5=82owski?= Date: Thu, 13 Sep 2012 11:09:35 +0200 Subject: db-update: Ignore repos in repos. --- db-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-update b/db-update index 875b87f..0359697 100755 --- a/db-update +++ b/db-update @@ -9,7 +9,7 @@ if [ $# -ge 1 ]; then fi # Find repos with packages to release -repos=($(find "${STAGING}" -mindepth 1 -type d ! -empty -printf '%f ' 2>/dev/null)) +repos=($(find "${STAGING}" -mindepth 1 -maxdepth 1 -type d ! -empty -printf '%f ' 2>/dev/null)) if [ $? -ge 1 ]; then die "Could not read ${STAGING}" fi -- cgit v1.2.3-2-g168b From d8ee98931359b87871bf0bef0b2ac5fca19d4d20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Tue, 2 Oct 2012 13:03:11 -0300 Subject: Deprecating repo-add with license extraction --- repo-add | 561 --------------------------------------------------------------- 1 file changed, 561 deletions(-) delete mode 100755 repo-add diff --git a/repo-add b/repo-add deleted file mode 100755 index ef9c1e1..0000000 --- a/repo-add +++ /dev/null @@ -1,561 +0,0 @@ -#!/bin/bash -# -# repo-add - add a package to a given repo database file -# repo-remove - remove a package entry from a given repo database file -# Generated from repo-add.in; do not edit by hand. -# -# Copyright (c) 2006-2008 Aaron Griffin -# Copyright (c) 2007-2008 Dan McGee -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -LICENSESDIR=/srv/http/repo/licenses - -# gettext initialization -export TEXTDOMAIN='pacman' -export TEXTDOMAINDIR='/usr/share/locale' - -myver='3.5.0' -confdir='/home/parabolavnx/etc' - -QUIET=0 -DELTA=0 -WITHFILES=0 -REPO_DB_FILE= -LOCKFILE= -CLEAN_LOCK=0 - -# ensure we have a sane umask set -umask 0022 - -msg() { - (( QUIET )) && return - local mesg=$1; shift - printf "==> ${mesg}\n" "$@" >&1 -} - -msg2() { - (( QUIET )) && return - local mesg=$1; shift - printf " -> ${mesg}\n" "$@" >&1 -} - -warning() { - local mesg=$1; shift - printf "==> $(gettext "WARNING:") ${mesg}\n" "$@" >&2 -} - -error() { - local mesg=$1; shift - printf "==> $(gettext "ERROR:") ${mesg}\n" "$@" >&2 -} - -# print usage instructions -usage() { - printf "repo-add, repo-remove (pacman) %s\n\n" "$myver" - printf "$(gettext "Usage: repo-add [-d] [-f] [-q] ...\n")" - printf "$(gettext "Usage: repo-remove [-q] ...\n\n")" - printf "$(gettext "\ -repo-add will update a package database by reading a package file.\n\ -Multiple packages to add can be specified on the command line.\n\n")" - printf "$(gettext "\ -repo-remove will update a package database by removing the package name\n\ -specified on the command line from the given repo database. Multiple\n\ -packages to remove can be specified on the command line.\n\n")" - printf "$(gettext "\ -Use the -q/--quiet flag to minimize output to basic messages, warnings,\n\ -and errors.\n\n")" - printf "$(gettext "\ -Use the -d/--delta flag to automatically generate and add a delta file\n\ -between the old entry and the new one, if the old package file is found\n\ -next to the new one.\n\n")" - printf "$(gettext "\ -Use the -f/--files flag to update a database including file entries.\n\n")" - echo "$(gettext "Example: repo-add /path/to/repo.db.tar.gz pacman-3.0.0.pkg.tar.gz")" - echo "$(gettext "Example: repo-remove /path/to/repo.db.tar.gz kernel26")" -} - -version() { - printf "repo-add, repo-remove (pacman) %s\n\n" "$myver" - printf "$(gettext "\ -Copyright (C) 2006-2008 Aaron Griffin .\n\ -Copyright (c) 2007-2008 Dan McGee .\n\n\ -This is free software; see the source for copying conditions.\n\ -There is NO WARRANTY, to the extent permitted by law.\n")" -} - -# write a list entry -# arg1 - Entry name -# arg2 - List -# arg3 - File to write to -write_list_entry() { - if [[ -n $2 ]]; then - echo "%$1%" >>$3 - echo -e $2 >>$3 - fi -} - -find_pkgentry() -{ - local pkgname=$1 - local pkgentry - for pkgentry in $tmpdir/$pkgname*; do - name=${pkgentry##*/} - if [[ ${name%-*-*} = $pkgname ]]; then - echo $pkgentry - return 0 - fi - done - return 1 -} - -# Get the package name from the delta filename -get_delta_pkgname() { - local tmp - - tmp=${1##*/} - echo ${tmp%-*-*_to*} -} - -# write a delta entry -# arg1 - path to delta file -db_write_delta() -{ - deltafile="$1" - pkgname="$(get_delta_pkgname $deltafile)" - - pkgentry=$(find_pkgentry $pkgname) - if [[ -z $pkgentry ]]; then - error "$(gettext "No database entry for package '%s'.")" "$pkgname" - return 1 - fi - deltas="$pkgentry/deltas" - if [[ ! -f $deltas ]]; then - echo -e "%DELTAS%" >$deltas - fi - # get md5sum and compressed size of package - md5sum="$(openssl dgst -md5 "$deltafile")" - md5sum="${md5sum##* }" - csize=$(stat -L -c %s "$deltafile") - - oldfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (source)" | sed 's/.*: *//') - newfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (output)" | sed 's/.*: *//') - - if grep -q "$oldfile.*$newfile" $deltas; then - sed -i.backup "/$oldfile.*$newfile/d" $deltas && rm -f $deltas.backup - fi - msg2 "$(gettext "Adding 'deltas' entry : %s -> %s")" "$oldfile" "$newfile" - echo ${deltafile##*/} $md5sum $csize $oldfile $newfile >> $deltas - - return 0 -} # end db_write_delta - -# remove a delta entry -# arg1 - path to delta file -db_remove_delta() -{ - deltafile="$1" - filename=${deltafile##*/} - pkgname="$(get_delta_pkgname $deltafile)" - - pkgentry=$(find_pkgentry $pkgname) - if [[ -z $pkgentry ]]; then - return 1 - fi - deltas="$pkgentry/deltas" - if [[ ! -f $deltas ]]; then - return 1 - fi - if grep -q "$filename" $deltas; then - sed -i.backup "/$filename/d" $deltas && rm -f $deltas.backup - msg2 "$(gettext "Removing existing entry '%s'...")" "$filename" - return 0 - fi - - return 1 -} # end db_remove_delta - -# write an entry to the pacman database -# arg1 - path to package -db_write_entry() -{ - # blank out all variables - local pkgfile="$1" - local pkgname pkgver pkgdesc csize size md5sum url arch builddate packager \ - _groups _licenses _replaces _depends _conflicts _provides _optdepends - - local OLDIFS="$IFS" - # IFS (field separator) is only the newline character - IFS=" -" - - # read info from the zipped package - local line var val - for line in $(bsdtar -xOqf "$pkgfile" .PKGINFO | - grep -v '^#' | sed 's|\(\w*\)\s*=\s*\(.*\)|\1 \2|'); do - # bash awesomeness here- var is always one word, val is everything else - var=${line%% *} - val=${line#* } - declare $var="$val" - case "$var" in - group) _groups="$_groups$group\n" ;; - license) _licenses="$_licenses$license\n" ;; - replaces) _replaces="$_replaces$replaces\n" ;; - depend) _depends="$_depends$depend\n" ;; - conflict) _conflicts="$_conflicts$conflict\n" ;; - provides) _provides="$_provides$provides\n" ;; - optdepend) _optdepends="$_optdepends$optdepend\n" ;; - esac - done - - IFS=$OLDIFS - - # get md5sum and compressed size of package - md5sum="$(openssl dgst -md5 "$pkgfile")" - md5sum="${md5sum##* }" - csize=$(stat -L -c %s "$pkgfile") - - # ensure $pkgname and $pkgver variables were found - if [[ -z $pkgname || -z $pkgver ]]; then - error "$(gettext "Invalid package file '%s'.")" "$pkgfile" - return 1 - fi - - pushd "$tmpdir" >/dev/null - if [[ -d $pkgname-$pkgver ]]; then - warning "$(gettext "An entry for '%s' already existed")" "$pkgname-$pkgver" - else - if (( DELTA )); then - pkgentry=$(find_pkgentry $pkgname) - if [[ -n $pkgentry ]]; then - local oldfilename=$(grep -A1 FILENAME $pkgentry/desc | tail -n1) - local oldfile="$(dirname $1)/$oldfilename" - fi - fi - fi - - # remove an existing entry if it exists, ignore failures - db_remove_entry "$pkgname" - - # create package directory - mkdir "$pkgname-$pkgver" - pushd "$pkgname-$pkgver" >/dev/null - - # restore an eventual deltas file - [[ -f ../$pkgname.deltas ]] && mv "../$pkgname.deltas" deltas - - # create desc entry - msg2 "$(gettext "Creating '%s' db entry...")" 'desc' - echo -e "%FILENAME%\n$(basename "$1")\n" >>desc - echo -e "%NAME%\n$pkgname\n" >>desc - [[ -n $pkgbase ]] && echo -e "%BASE%\n$pkgbase\n" >>desc - echo -e "%VERSION%\n$pkgver\n" >>desc - [[ -n $pkgdesc ]] && echo -e "%DESC%\n$pkgdesc\n" >>desc - write_list_entry "GROUPS" "$_groups" "desc" - [[ -n $csize ]] && echo -e "%CSIZE%\n$csize\n" >>desc - [[ -n $size ]] && echo -e "%ISIZE%\n$size\n" >>desc - - # compute checksums - msg2 "$(gettext "Computing md5 checksums...")" - echo -e "%MD5SUM%\n$md5sum\n" >>desc - - [[ -n $url ]] && echo -e "%URL%\n$url\n" >>desc - write_list_entry "LICENSE" "$_licenses" "desc" - [[ -n $arch ]] && echo -e "%ARCH%\n$arch\n" >>desc - [[ -n $builddate ]] && echo -e "%BUILDDATE%\n$builddate\n" >>desc - [[ -n $packager ]] && echo -e "%PACKAGER%\n$packager\n" >>desc - write_list_entry "REPLACES" "$_replaces" "desc" - - # create depends entry - msg2 "$(gettext "Creating '%s' db entry...")" 'depends' - # create the file even if it will remain empty - touch "depends" - write_list_entry "DEPENDS" "$_depends" "depends" - write_list_entry "CONFLICTS" "$_conflicts" "depends" - write_list_entry "PROVIDES" "$_provides" "depends" - write_list_entry "OPTDEPENDS" "$_optdepends" "depends" - - popd >/dev/null - popd >/dev/null - - # create files file if wanted - if (( WITHFILES )); then - msg2 "$(gettext "Creating '%s' db entry...")" 'files' - local files_path="$tmpdir/$pkgname-$pkgver/files" - echo "%FILES%" >$files_path - bsdtar --exclude='.*' -tf "$pkgfile" >>$files_path - fi - - # create a delta file - if (( DELTA )); then - if [[ -n $oldfilename ]]; then - if [[ -f $oldfile ]]; then - delta=$(pkgdelta -q $oldfile $1) - if [[ -f $delta ]]; then - db_write_delta $delta - fi - else - warning "$(gettext "Old package file not found: %s")" "$oldfilename" - fi - fi - fi - - # Extracts licenses to a common license dir - msg "Extracting license" - if bsdtar -xOf ${pkgfile} .PKGINFO | grep "license" | grep "custom" ; then - if [ -d ${LICENSESDIR}/${pkgname} ]; then - rm -r ${LICENSESDIR}/${pkgname} - fi - - # Change dir to licenses, and extract them stripping the first part of the path - bsdtar -C ${LICENSESDIR}/ --include="usr/share/licenses/" \ - --strip-components 3 -xf ${pkgfile} >/dev/null 2>&1 - - if [ $? -ne 0 ]; then - warning "This package doesn't contain a license dir" - fi - fi - - return 0 -} # end db_write_entry - -# remove existing entries from the DB -# arg1 - package name -db_remove_entry() { - local pkgname=$1 - local notfound=1 - local pkgentry=$(find_pkgentry $pkgname) - while [[ -n $pkgentry ]]; do - notfound=0 - if [[ -f $pkgentry/deltas ]]; then - mv "$pkgentry/deltas" "$tmpdir/$pkgname.deltas" - fi - msg2 "$(gettext "Removing existing entry '%s'...")" \ - "$(basename $pkgentry)" - rm -rf $pkgentry - pkgentry=$(find_pkgentry $pkgname) - done - - msg "Removing license" - if [ -d ${LICENSESDIR}/${pkgname} ]; then - rm -r ${LICENSESDIR}/${pkgname} - fi - - return $notfound -} # end db_remove_entry - -check_repo_db() -{ - # check lock file - if ( set -o noclobber; echo "$$" > "$LOCKFILE") 2> /dev/null; then - CLEAN_LOCK=1 - else - error "$(gettext "Failed to acquire lockfile: %s.")" "$LOCKFILE" - [[ -f $LOCKFILE ]] && error "$(gettext "Held by process %s")" "$(cat $LOCKFILE)" - exit 1 - fi - - if [[ -f $REPO_DB_FILE ]]; then - # there are two situations we can have here- a DB with some entries, - # or a DB with no contents at all. - if ! bsdtar -tqf "$REPO_DB_FILE" '*/desc' >/dev/null 2>&1; then - # check empty case - if [[ -n $(bsdtar -tqf "$REPO_DB_FILE" '*' 2>/dev/null) ]]; then - error "$(gettext "Repository file '%s' is not a proper pacman database.")" "$REPO_DB_FILE" - exit 1 - fi - fi - msg "$(gettext "Extracting database to a temporary location...")" - bsdtar -xf "$REPO_DB_FILE" -C "$tmpdir" - else - case "$cmd" in - repo-remove) - error "$(gettext "Repository file '%s' was not found.")" "$REPO_DB_FILE" - exit 1 - ;; - repo-add) - # check if the file can be created (write permission, directory existence, etc) - if ! touch "$REPO_DB_FILE"; then - error "$(gettext "Repository file '%s' could not be created.")" "$REPO_DB_FILE" - exit 1 - fi - rm -f "$REPO_DB_FILE" - ;; - esac - fi -} - -add() -{ - if [[ ! -f $1 ]]; then - error "$(gettext "File '%s' not found.")" "$1" - return 1 - fi - - if [[ ${1##*.} == "delta" ]]; then - deltafile=$1 - msg "$(gettext "Adding delta '%s'")" "$deltafile" - if ! type xdelta3 &>/dev/null; then - error "$(gettext "Cannot find the xdelta3 binary! Is xdelta3 installed?")" - exit 1 - fi - if db_write_delta "$deltafile"; then - return 0 - else - return 1 - fi - fi - - pkgfile=$1 - if ! bsdtar -tqf "$pkgfile" .PKGINFO >/dev/null 2>&1; then - error "$(gettext "'%s' is not a package file, skipping")" "$pkgfile" - return 1 - fi - - msg "$(gettext "Adding package '%s'")" "$pkgfile" - - db_write_entry "$pkgfile" -} - -remove() -{ - if [[ ${1##*.} == "delta" ]]; then - deltafile=$1 - msg "$(gettext "Searching for delta '%s'...")" "$deltafile" - if db_remove_delta "$deltafile"; then - return 0 - else - error "$(gettext "Delta matching '%s' not found.")" "$deltafile" - return 1 - fi - fi - - pkgname=$1 - msg "$(gettext "Searching for package '%s'...")" "$pkgname" - - if db_remove_entry "$pkgname"; then - rm -f "$tmpdir/$pkgname.deltas" - return 0 - else - error "$(gettext "Package matching '%s' not found.")" "$pkgname" - return 1 - fi -} - -trap_exit() -{ - echo - error "$@" - exit 1 -} - -clean_up() { - local exit_code=$? - - [[ -d $tmpdir ]] && rm -rf "$tmpdir" - (( CLEAN_LOCK )) && [[ -f $LOCKFILE ]] && rm -f "$LOCKFILE" - - exit $exit_code -} - -# PROGRAM START - -# determine whether we have gettext; make it a no-op if we do not -if ! type gettext &>/dev/null; then - gettext() { - echo "$@" - } -fi - -case "$1" in - -h|--help) usage; exit 0;; - -V|--version) version; exit 0;; -esac - -# figure out what program we are -cmd="$(basename $0)" -if [[ $cmd != "repo-add" && $cmd != "repo-remove" ]]; then - error "$(gettext "Invalid command name '%s' specified.")" "$cmd" - exit 1 -fi - -tmpdir=$(mktemp -d /tmp/repo-tools.XXXXXXXXXX) || (\ - error "$(gettext "Cannot create temp directory for database building.")"; \ - exit 1) - -trap 'clean_up' EXIT -trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT -trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT -trap 'trap_exit "$(gettext "An unknown error has occured. Exiting...")"' ERR - -success=0 -# parse arguments -for arg in "$@"; do - case "$arg" in - -q|--quiet) QUIET=1;; - -d|--delta) DELTA=1;; - -f|--files) WITHFILES=1;; - *) - if [[ -z $REPO_DB_FILE ]]; then - REPO_DB_FILE="$arg" - LOCKFILE="$REPO_DB_FILE.lck" - check_repo_db - else - case "$cmd" in - repo-add) add $arg && success=1 ;; - repo-remove) remove $arg && success=1 ;; - esac - fi - ;; - esac -done - -# if at least one operation was a success, re-zip database -if (( success )); then - msg "$(gettext "Creating updated database file '%s'")" "$REPO_DB_FILE" - - case "$REPO_DB_FILE" in - *tar.gz) TAR_OPT="z" ;; - *tar.bz2) TAR_OPT="j" ;; - *tar.xz) TAR_OPT="J" ;; - *) warning "$(gettext "'%s' does not have a valid archive extension.")" \ - "$REPO_DB_FILE" ;; - esac - - filename=$(basename "$REPO_DB_FILE") - - pushd "$tmpdir" >/dev/null - if [[ -n $(ls) ]]; then - bsdtar -c${TAR_OPT}f "$filename" * - else - # we have no packages remaining? zip up some emptyness - warning "$(gettext "No packages remain, creating empty database.")" - bsdtar -c${TAR_OPT}f "$filename" -T /dev/null - fi - popd >/dev/null - - [[ -f $REPO_DB_FILE ]] && mv -f "$REPO_DB_FILE" "${REPO_DB_FILE}.old" - [[ -f $tmpdir/$filename ]] && mv "$tmpdir/$filename" "$REPO_DB_FILE" - dblink="${REPO_DB_FILE%.tar.*}" - target=${REPO_DB_FILE##*/} - ln -sf "$target" "$dblink" 2>/dev/null || \ - ln -f "$target" "$dblink" 2>/dev/null || \ - cp "$REPO_DB_FILE" "$dblink" -else - msg "$(gettext "No packages modified, nothing to do.")" - exit 1 -fi - -exit 0 -# vim: set ts=2 sw=2 noet: -- cgit v1.2.3-2-g168b From 53eb3d26ed11098b0d1803dcee46bef40c1c67d8 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 7 Oct 2012 18:52:52 +0000 Subject: Use system repo-add --- any-to-ours | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/any-to-ours b/any-to-ours index 3a58946..a1d6686 100755 --- a/any-to-ours +++ b/any-to-ours @@ -62,8 +62,8 @@ for _repo in ${ARCHREPOS[@]}; do pushd ${FTP_BASE}/${_repo}/os/${_arch}/ >/dev/null # Add the packages to the db - $(dirname $0)/repo-add ${_repo}${DBEXT} \ - ${SYNCED[@]} + repo-add ${_repo}${DBEXT} \ + ${SYNCED[@]} popd >/dev/null -- cgit v1.2.3-2-g168b From 58dcd2bd938eb64e3c0d31841f98f43a8a1ca346 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sat, 13 Oct 2012 10:57:28 -0300 Subject: Deprecating repo-update Thanks to aurelien to make me find the misterious garbage packages under staging/! --- repo-update | 59 ----------------------------------------------------------- 1 file changed, 59 deletions(-) delete mode 100755 repo-update diff --git a/repo-update b/repo-update deleted file mode 100755 index bf8d73c..0000000 --- a/repo-update +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -# -*- coding: utf-8 -*- -source ~/.bashrc -source $(dirname $0)/config -source $(dirname $0)/local_config -source $(dirname $0)/libremessages - -for repo in ${ARCHREPOS[@]}; do - - msg "Syncing ${repo}" - for arch in ${ARCHARCHES[@]}; do - msg2 "${repo} ${arch}" - # makes a file containing rsync output for filter.py - ${rsync_list_command} \ - rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ - ${repodir}/staging/${repo}/os/${arch}/ > ${rsout_file}-${repo}-${arch} || exit 1 - # reads blacklist and rsout_file and makes an rsync exclude-from - # list - filter.py -r ${rsync_blacklist} -k ${blacklist} \ - -f ${rsout_file}-${repo}-${arch} || exit 1 - # list files in ${repodir}/${repo} and write their names on - # rsync_not_needed for using as an rsync exclude-from - #find ${repodir}/${repo}/os/${arch}/ -name "*${PKGEXT}" \ - # -fprintf ${rsync_not_needed}-${repo}-${arch} '%f\n' || exit 1 - #find ${repodir}/${PKGPOOL}/ -name "*${PKGEXT}" \ - find ${repodir}/${repo}/os/${arch}/ -name "*${PKGEXT}" \ - -fprintf ${rsync_not_needed}-${repo}-${arch} '%f\n' || exit 1 - # Actual rsync command - ${rsync_update_command} \ - --exclude-from=${rsync_blacklist} \ - --exclude-from=${rsync_not_needed}-${repo}-${arch} \ - rsync://${mirror}/${mirrorpath}/${repo}/os/${arch}/ \ - ${repodir}/staging/${repo}/ || exit 1 - done - for arch in ${ARCHARCHES[@]}; do - msg2 "Making pending list for $repo $arch" - # if there is a db in repo (db is created on rsync) - if [ -r ${repodir}/staging/${repo}/os/${arch}/${repo}${DBEXT} ]; then - # clean_repo makes pending list with files on db and remove - # packages from db - $(dirname $0)/clean_repo.py -k ${blacklist} -w ${whitelist} \ - -p ${docs_dir}/pending-${repo}.txt \ - -b ${repodir}/staging/${repo}/${repo}${DBEXT} \ - -d ${repodir}/stagging/${repo} - fi - done - # if some nonfree files got pass the filter this command delete them - msg2 "Fallback cleaning repo" - $(dirname $0)/clean_repo.py -k ${blacklist} -d ${repodir}/staging/${repo} || exit 1 -done - -msg "Removing leftover files..." -find ${repodir}/staging/ -type f \! -name "*${PKGEXT}" -delete -# Staging should not have symbolic links -find ${repodir}/staging/ -type l -delete - -$(dirname $0)/db-update -$(dirname $0)/db-check-nonfree - -- cgit v1.2.3-2-g168b From f516cdb29d89027d267b3a09c98e518e7177251a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sat, 13 Oct 2012 11:13:25 -0300 Subject: Deprecating old python scripts --- __init__.py | 0 clean_repo.py | 99 ---------------------------- config.py | 68 -------------------- filter.py | 204 ---------------------------------------------------------- 4 files changed, 371 deletions(-) delete mode 100644 __init__.py delete mode 100755 clean_repo.py delete mode 100755 config.py delete mode 100755 filter.py diff --git a/__init__.py b/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/clean_repo.py b/clean_repo.py deleted file mode 100755 index 076de60..0000000 --- a/clean_repo.py +++ /dev/null @@ -1,99 +0,0 @@ -#! /usr/bin/python2 -#-*- encoding: utf-8 -*- -from filter import * -import argparse - -def mkpending(packages_iterable, pending_file, blacklisted_names, - whitelisted_names): - """ Determine wich packages are pending for license auditing.""" - search = tuple(blacklisted_names + - whitelisted_names) - - try: - fsock=open(pending_file, "r") - pkgs=[pkg for pkg in packages_iterable - if pkg["name"] not in listado(pending_file)] - for line in fsock.readlines(): - if line: - pkg=Package() - pkg["name"]=line.split(":")[0] - pkg["license"]=":".join(line.split(":")[1:]) - pkgs.append(pkg) - pkgs=[pkg for pkg in pkgs if pkg["name"] not in search - and "custom" in pkg["license"]] - fsock=open(pending_file, "w") - fsock.write("\n".join([pkg["name"] + ":" + pkg["location"] + - ":" + pkg["license"] - for pkg in pkgs]) + "\n") - fsock.close() - except(IOError): - printf("Can't read or write %s" % pending_file) - return pkgs - -def remove_from_blacklist(path_to_db, blacklisted_names): - """ Check the blacklist and remove packages on the db""" - if "~" in path_to_db: - path_to_db=(os.path.expanduser(path_to_db)) - - pkgs=[pkg for pkg in pkginfo_from_db(path_to_db) if - pkg["name"] in blacklisted_names] - if pkgs: - lista=" ".join(pkgs) - cmd = "repo-remove " + path_to_db + " " + lista - printf(cmd) - a = check_output(cmd) - return pkgs - -def cleanup_nonfree_in_dir(directory, blacklisted_names): - if "~" in directory: - directory=(os.path.expanduser(directory)) - pkglist=list() - pkgs=pkginfo_from_files_in_dir(directory) - for package in pkgs: - if package["name"] in blacklisted_names: - os.remove(package["location"]) - pkglist.append(package) - return pkglist - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - prog="clean_repo", - description="Clean a repo db and packages",) - - parser.add_argument("-k", "--blacklist-file", type=str, - help="File containing blacklisted names", - required=True,) - - group_dir=parser.add_argument_group("Clean non-free packages in dir") - group_dir.add_argument("-d", "--directory", type=str, - help="directory to clean") - - group_db=parser.add_argument_group("Clean non-free packages in db", - "All these arguments need to be specified for db cleaning:") - group_db.add_argument("-b", "--database", type=str, - help="dabatase to clean") - group_db.add_argument("-p", "--pending-file", type=str, - help="File in which to write pending list") - group_db.add_argument("-w", "--whitelist-file", type=str, - help="File containing whitelisted names") - - args=parser.parse_args() - - if args.database and not (args.pending_file and args.whitelist_file): - parser.print_help() - exit(1) - - blacklisted=listado(args.blacklist_file) - - if args.database: - whitelisted=listado(args.whitelist_file) - pkgs=pkginfo_from_db(args.database) - pending_names=[pkg["name"] for pkg in - mkpending(pkgs, args.pending_file, - blacklisted, whitelisted)] - - if args.directory and args.database: - cleanup_nonfree_in_dir(args.directory, (blacklisted + pending_names)) - elif args.directory: - cleanup_nonfree_in_dir(args.directory, blacklisted) - diff --git a/config.py b/config.py deleted file mode 100755 index 0ffc475..0000000 --- a/config.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/python2 -# -*- coding: utf-8 -*- -try: - from subprocess import check_output -except(ImportError): - from commands import getoutput - def check_output(*popenargs,**kwargs): - cmd=" ".join(*popenargs) - return getoutput(cmd) -import os - - -# Rsync commands - -def printf(text, logfile=False): - """Guarda el texto en la variable log y puede imprimir en pantalla.""" - print (str(text) + "\n") - if logfile: - try: - log = open(logfile, 'a') - log.write("\n" + str(text) + "\n") - except: - print("Can't open %s" % logfile) - finally: - log.close() - - -# Classes and Exceptions -class NonValidFile(ValueError): pass -class NonValidDir(ValueError): pass -class NonValidCommand(ValueError): pass - -class Package: - """ An object that has information about a package. """ - package_info=dict() - - def __init__(self): - self.package_info={ "name" : False, - "version" : False, - "release" : False, - "arch" : False, - "license" : False, - "location": False, - "depends" : False,} - - def __setitem__(self, key, item): - if key in self.package_info.keys(): - return self.package_info.__setitem__(key, item) - else: - raise ValueError("Package has no %s attribute" % key) - - def __getitem__(self, key): - return self.package_info.__getitem__(key) - - def __unicode__(self): - return str(self.package_info) - - def __repr__(self): - return str(self.package_info) - - def __eq__(self,x): - if not isinstance(x, Package): - return False - for key in self.package_info.keys(): - if x[key] != self.package_info[key]: - return False - else: - return True diff --git a/filter.py b/filter.py deleted file mode 100755 index 70d5e29..0000000 --- a/filter.py +++ /dev/null @@ -1,204 +0,0 @@ -#! /usr/bin/python2 -#-*- encoding: utf-8 -*- -from glob import glob -from config import * -import tarfile - -def listado(filename, start=0, end=None): - """Obtiene una lista de paquetes de un archivo.""" - fsock = open(filename, "r") - lista = fsock.read().split("\n") - fsock.close() - if end is not None: - return [pkg.split(":")[start:end].rstrip() - for pkg in lista if pkg] - else: - return [pkg.split(":")[start].rstrip() - for pkg in lista if pkg] - -def pkginfo_from_filename(filename): - """ Generates a Package object with info from a filename, - filename can be relative or absolute - - Parameters: - ---------- - filename -> str Must contain .pkg.tar. - - Returns: - ---------- - pkg -> Package object""" - if ".pkg.tar." not in filename: - raise NonValidFile("File is not a pacman package") - pkg = Package() - pkg["location"] = filename - fileattrs = os.path.basename(filename).split("-") - pkg["arch"] = fileattrs.pop(-1).split(".")[0] - pkg["release"] = fileattrs.pop(-1) - pkg["version"] = fileattrs.pop(-1) - pkg["name"] = "-".join(fileattrs) - return pkg - -def pkginfo_from_desc(info_from_desc, pkg=Package()): - """ Returns pkginfo from desc file. - - Parameters: - ---------- - filename -> str File must exist - - Returns: - ---------- - pkg -> Package object""" - info=info_from_desc.rsplit() - info_map={"name" :("%NAME%" , None), - "version" :("%VERSION%" , 0 ), - "release" :("%VERSION%" , 1 ), - "arch" :("%ARCH%" , None), - "license" :("%LICENSE%" , None), - "location":("%FILENAME%", None),} - - for key in info_map.keys(): - field,pos=info_map[key] - pkg[key]=info[info.index(field)+1] - if pos is not None: - pkg[key]=pkg[key].split("-")[pos] - return pkg - -def pkginfo_from_rsync_output(rsync_output): - """ Generates a list of packages and versions from an rsync output - wich uses --list-only and --no-motd options. - - Parameters: - ---------- - rsync_output -> str Contains output from rsync - - Returns: - ---------- - package_list -> tuple Contains Package objects. """ - - def package_or_link(line): - """ Take info out of filename """ - location_field = 4 - return pkginfo_from_filename(line.rsplit()[location_field]) - - def do_nothing(): - pass - - options = { "d": do_nothing, - "l": package_or_link, - "-": package_or_link, - " ": do_nothing} - - package_list=list() - - lines=[x for x in rsync_output.split("\n") if ".pkg.tar" in x] - - for line in lines: - pkginfo=options[line[0]](line) - if pkginfo: - package_list.append(pkginfo) - - return tuple(package_list) - -def pkginfo_from_files_in_dir(directory): - """ Returns pkginfo from filenames of packages in dir - wich has .pkg.tar. on them - - Parameters: - ---------- - directory -> str Directory must exist - - Returns: - ---------- - package_list -> tuple Contains Package objects """ - package_list=list() - - if not os.path.isdir(directory): - raise NonValidDir - - for filename in glob(os.path.join(directory,"*")): - if ".pkg.tar." in filename: - package_list.append(pkginfo_from_filename(filename)) - return tuple(package_list) - -def pkginfo_from_db(path_to_db): - """ Get pkginfo from db. - - Parameters: - ---------- - path_to_db -> str Path to file - - Output: - ---------- - package_list -> tuple of Package objects""" - package_list=list() - - if not os.path.isfile(path_to_db): - raise NonValidFile(path_to_db + " is not a file") - - try: - dbsock = tarfile.open(path_to_db, 'r:gz') - desc_files=[desc for desc in dbsock.getnames() - if "/desc" in desc] - for name in desc_files: - desc=dbsock.extractfile(name).read().decode("UTF-8") - package_list.append(pkginfo_from_desc(desc)) - except tarfile.ReadError: - raise NonValidFile("No valid db_file %s or not readable" - % path_to_db) - finally: - dbsock.close() - return package_list - -def rsyncBlacklist_from_blacklist(packages_iterable, - blacklisted_names, - exclude_file): - """ Generate an exclude list for rsync - - Parameters: - ---------- - package_iterable -> list or tuple Contains Package objects - blacklisted_names-> list or tuple Contains blacklisted names - exclude_file -> str Path to file - debug -> bool If True, file list gets logged - - Output: - ---------- - None """ - pkgs=[pkg["location"] for pkg in packages_iterable - if isinstance(pkg, Package) - and pkg["name"] in blacklisted_names] - if exclude_file: - try: - fsock = open(exclude_file,"w") - fsock.write("\n".join(pkgs) + "\n") - except IOError: - printf("%s wasnt written" % exclude_file) - exit(1) - finally: - fsock.close() - return pkgs - - -if __name__ == "__main__": - import argparse - parser=argparse.ArgumentParser() - parser.add_argument("-r", "--rsync-exclude-file", type=str, - help="File in which to generate exclude list", - required=True,) - parser.add_argument("-k", "--blacklist-file", type=str, - help="File containing blacklisted names", - required=True,) - parser.add_argument("-f", "--rsout-file", type=str, - help="This file will be read to get a pkg list", - required=True,) - args=parser.parse_args() - try: - fsock=open(args.rsout_file, "r") - rsout=fsock.read() - except IOError: - print("%s is not readable" % args.rsout_file) - finally: - fsock.close() - packages=pkginfo_from_rsync_output(rsout) - rsyncBlacklist_from_blacklist(packages, listado(args.blacklist_file), - args.rsync_exclude_file) -- cgit v1.2.3-2-g168b From f4c42f9e68131ad4fbfb1be07fb44678eb90d6f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Mas=C5=82owski?= Date: Sat, 13 Oct 2012 18:06:52 +0200 Subject: db-list-unsigned-packages: New script. --- db-list-unsigned-packages | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100755 db-list-unsigned-packages diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages new file mode 100755 index 0000000..35a5421 --- /dev/null +++ b/db-list-unsigned-packages @@ -0,0 +1,47 @@ +#!/bin/bash +# Copyright (C) 2012 Michał Masłowski +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -e + +# Output a list of repo/package-name-and-version pairs representing +# unsigned packages available for architecture $1 and specified for +# architecture $2 (usually $1 or any, default is to list all). + +. "$(dirname $0)/db-functions" +. "$(dirname $0)/config" + +if [ $# -lt 1 ]; then + msg "usage: $(basename $0) " + exit 1 +fi + +arch=$1 +pkgarch=$2 + +for repo in ${PKGREPOS[@]} +do + db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" + for f in $(tar tf $db | egrep /desc$) + do + if ! tar xOf $db $f | fgrep %PGPSIG% > /dev/null + then + if [ -z $2 ] || tar xOf $db $f | fgrep -A1 %ARCH% | fgrep $pkgarch > /dev/null + then + echo $repo/${f%/desc} + fi + fi + done +done -- cgit v1.2.3-2-g168b From c6542576f04e6dc159e731b50a19f1a10d9d5ff3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Mas=C5=82owski?= Date: Tue, 16 Oct 2012 15:42:10 +0200 Subject: db-list-unsigned-packages: rewrite using a helper Python script. The previous implementation parsed each tarball multiple times having quadratic time complexity in the number of packages. It was too slow for a complete run. --- db-list-unsigned-packages | 13 ++-------- db-list-unsigned-packages.py | 59 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 11 deletions(-) create mode 100755 db-list-unsigned-packages.py diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages index 35a5421..26e22eb 100755 --- a/db-list-unsigned-packages +++ b/db-list-unsigned-packages @@ -29,19 +29,10 @@ if [ $# -lt 1 ]; then fi arch=$1 -pkgarch=$2 +shift for repo in ${PKGREPOS[@]} do db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" - for f in $(tar tf $db | egrep /desc$) - do - if ! tar xOf $db $f | fgrep %PGPSIG% > /dev/null - then - if [ -z $2 ] || tar xOf $db $f | fgrep -A1 %ARCH% | fgrep $pkgarch > /dev/null - then - echo $repo/${f%/desc} - fi - fi - done + "$(dirname $0)/db-list-unsigned-packages.py" "$@" < "$db" done diff --git a/db-list-unsigned-packages.py b/db-list-unsigned-packages.py new file mode 100755 index 0000000..ccbec3a --- /dev/null +++ b/db-list-unsigned-packages.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# Copyright (C) 2012 Michał Masłowski +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +Output a list of repo/package-name-and-version pairs representing +unsigned packages in the database at standard input of repo named in +the first argument and specified for architectures listed in the +following arguments (usually the one of the database or any, default +is to list all). +""" + + +import sys +import tarfile + + +def main(): + """Do the job.""" + repo = sys.argv[1] + pkgarches = frozenset(name.encode("utf-8") for name in sys.argv[2:]) + with tarfile.open(fileobj=sys.stdin.buffer) as archive: + for entry in archive: + if entry.name.endswith("/desc"): + content = archive.extractfile(entry) + skip = False + is_arch = False + for line in content: + if is_arch: + is_arch = False + if pkgarches and line.strip() not in pkgarches: + skip = True # different architecture + break + if line == b"%PGPSIG%\n": + skip = True # signed + break + if line == b"%ARCH%\n": + is_arch = True + if skip: + print("skip " + repo + "/" + entry.name[:-5]) + continue + print(repo + "/" + entry.name[:-5]) + + +if __name__ == "__main__": + main() -- cgit v1.2.3-2-g168b From be6cff9570eb4e55cfef7a8a83ffd65c37aa2c91 Mon Sep 17 00:00:00 2001 From: Parabola Date: Tue, 16 Oct 2012 13:53:57 +0000 Subject: db-list-unsigned-packages: Ignore missing packages, pass the repo name. --- db-list-unsigned-packages | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages index 26e22eb..3b5a5bd 100755 --- a/db-list-unsigned-packages +++ b/db-list-unsigned-packages @@ -34,5 +34,5 @@ shift for repo in ${PKGREPOS[@]} do db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" - "$(dirname $0)/db-list-unsigned-packages.py" "$@" < "$db" + [ -f "$db" ] && "$(dirname $0)/db-list-unsigned-packages.py" "$repo" "$@" < "$db" done -- cgit v1.2.3-2-g168b From 43aacb891613909fdb0e2898fc9d85bfa20bcfad Mon Sep 17 00:00:00 2001 From: Parabola Date: Tue, 16 Oct 2012 13:54:48 +0000 Subject: db-list-unsigned-packages.py: Don't list signed packages. --- db-list-unsigned-packages.py | 1 - 1 file changed, 1 deletion(-) diff --git a/db-list-unsigned-packages.py b/db-list-unsigned-packages.py index ccbec3a..36be93a 100755 --- a/db-list-unsigned-packages.py +++ b/db-list-unsigned-packages.py @@ -50,7 +50,6 @@ def main(): if line == b"%ARCH%\n": is_arch = True if skip: - print("skip " + repo + "/" + entry.name[:-5]) continue print(repo + "/" + entry.name[:-5]) -- cgit v1.2.3-2-g168b From 7a2f916a360b16c3649622d9f04c6424aab8f32d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sat, 20 Oct 2012 16:25:07 -0300 Subject: Remove broken symlinks after cleanup --- db-cleanup | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/db-cleanup b/db-cleanup index 72790f1..904c06e 100755 --- a/db-cleanup +++ b/db-cleanup @@ -60,4 +60,8 @@ for POOL in ${PKGPOOLS[@]} ${SRCPOOLS[@]}; do ${FTP_BASE}/${POOL}/ done +msg "Removing symlinks:" +find -L ${FTP_BASE}/ -type l +${CLEANUP_DRYRUN} || find -L ${FTP_BASE}/ -type l -delete + exit $? -- cgit v1.2.3-2-g168b From f8f54ad1926d3f10e895fda553474db278c2ed7d Mon Sep 17 00:00:00 2001 From: Parabola Date: Wed, 31 Oct 2012 04:44:23 +0000 Subject: sitting on the server --- README | 3 +++ config | 2 +- config.orig | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 README create mode 100644 config.orig diff --git a/README b/README new file mode 100644 index 0000000..9ce2d3f --- /dev/null +++ b/README @@ -0,0 +1,3 @@ +* Script to run if there is error and for update + + - db-sync diff --git a/config b/config index 20ad307..7601527 100644 --- a/config +++ b/config @@ -10,7 +10,7 @@ OURREPOS=('libre' 'libre-testing') # User repos USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde') # Community project repos -PROJREPOS=('pcr' 'social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross') +PROJREPOS=('pcr' 'social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross' 'java' 'java-ugly') # Remote repos RMTREPOS=('connos' 'connos-extra') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) diff --git a/config.orig b/config.orig new file mode 100644 index 0000000..a32f82f --- /dev/null +++ b/config.orig @@ -0,0 +1,54 @@ +#!/bin/bash +<<<<<<< HEAD +FTP_BASE="/srv/http/repo/public" +ARCH_BASE="/srv/http/repo/public" +SVNREPO="/srv/http/repo/abslibre" +======= +FTP_BASE="/srv/http/repo/public/temprepo" +ARCH_BASE="/srv/http/repo/public/temprepo" +SVNREPO="/var/abs" +>>>>>>> 801ea2c927ace5ee892209dd8e3c1044e1b3842e + +# Repos from Arch +ARCHREPOS=('core' 'testing') #'extra' 'community' 'testing' 'multilib') +# Official Parabola repos +OURREPOS=('libre' 'libre-testing') +# User repos +USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan') +# Community project repos +PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'sugar') +PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) +PKGPOOL='pool/packages' +SRCPOOL='sources/packages' + +CLEANUP_DESTDIR="$FTP_BASE/old/packages" +CLEANUP_DRYRUN=true +# Time in days to keep moved packages +CLEANUP_KEEP=30 + +SOURCE_CLEANUP_DESTDIR="$FTP_BASE/old/sources" +SOURCE_CLEANUP_DRYRUN=true +# Time in days to keep moved sourcepackages +SOURCE_CLEANUP_KEEP=30 + +REQUIRE_SIGNATURE=true + +LOCK_DELAY=10 +LOCK_TIMEOUT=300 + +STAGING="$FTP_BASE/staging" +TMPDIR="/tmp" +ARCHARCHES=(i686 x86_64) +OURARCHES=(mips64el) +ARCHES=(${ARCHARCHES[@]} ${OURARCHES[@]}) +DBEXT=".db.tar.gz" +FILESEXT=".files.tar.gz" +PKGEXT=".pkg.tar.?z" +SRCEXT=".src.tar.gz" + +<<<<<<< HEAD +MAKEPKGCONF="~/.makepkg.conf" +======= +MAKEPKGCONF="/etc/makepkg.conf" +>>>>>>> 801ea2c927ace5ee892209dd8e3c1044e1b3842e +BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" -- cgit v1.2.3-2-g168b From 0d409373d58f1e2ab8a6df6f2123b986aed562a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Mas=C5=82owski?= Date: Tue, 13 Nov 2012 22:35:07 +0100 Subject: get-repos: Don't stop after reporead fails. Issue #220 prevents it from working for some repos. --- get-repos | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/get-repos b/get-repos index 86b371c..4397d54 100755 --- a/get-repos +++ b/get-repos @@ -60,7 +60,7 @@ find "${TMPDIR}" -iname "*${DBEXT}" | while read _db; do continue fi - "${WEB_DIR}"/manage.py reporead "${_arch}" "${_db}" + "${WEB_DIR}"/manage.py reporead "${_arch}" "${_db}" || true done rm -r ${TMPDIR} -- cgit v1.2.3-2-g168b From 9fba20c1e3a70d5cb6e2a7f59d026ce3e28185e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Tue, 30 Apr 2013 14:11:10 -0300 Subject: Removed svn, should work with regular abs --- db-move | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/db-move b/db-move index 019faa2..010d941 100755 --- a/db-move +++ b/db-move @@ -24,12 +24,10 @@ for pkgarch in ${ARCHES[@]}; do repo_lock ${repo_from} ${pkgarch} || exit 1 done -# check if packages to be moved exist in svn and ftp dir -/usr/bin/svn checkout -q -N "${SVNREPO}" "${WORKDIR}/svn" >/dev/null +# No idea why we loop twice... -- fauno for pkgbase in ${args[@]:2}; do - /usr/bin/svn up -q "${WORKDIR}/svn/${pkgbase}" >/dev/null for pkgarch in ${ARCHES[@]} 'any'; do - svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" + svnrepo_from="${SVNREPO}/${repo_from}/${pkgbase}" if [ -r "${svnrepo_from}/PKGBUILD" ]; then pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo ${pkgname[@]})) if [ ${#pkgnames[@]} -lt 1 ]; then @@ -64,8 +62,7 @@ declare -A add_pkgs declare -A remove_pkgs for pkgbase in ${args[@]:2}; do for pkgarch in ${ARCHES[@]} 'any'; do - svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" - svnrepo_to="${WORKDIR}/svn/${pkgbase}/repos/${repo_to}-${pkgarch}" + svnrepo_from="${SVNREPO}/${repo_from}/${pkgbase}" if [ -f "${svnrepo_from}/PKGBUILD" ]; then if [ "${pkgarch}" == 'any' ]; then @@ -77,21 +74,6 @@ for pkgbase in ${args[@]:2}; do pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo ${pkgname[@]})) pkgver=$(. "${svnrepo_from}/PKGBUILD"; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) - if [ -d "${svnrepo_to}" ]; then - for file in $(/usr/bin/svn ls "${svnrepo_to}"); do - /usr/bin/svn rm -q "${svnrepo_to}/$file" - done - else - mkdir "${svnrepo_to}" - /usr/bin/svn add -q "${svnrepo_to}" - fi - - for file in $(svn ls "${svnrepo_from}"); do - /usr/bin/svn mv -q -r HEAD "${svnrepo_from}/$file" "${svnrepo_to}/" - done - /usr/bin/svn rm --force -q "${svnrepo_from}" - /usr/bin/svn commit -q "${WORKDIR}/svn/${pkgbase}" -m "$(basename $0): moved ${pkgbase} from [${repo_from}] to [${repo_to}] (${pkgarch})" - for pkgname in ${pkgnames[@]}; do for tarch in ${tarches[@]}; do pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT}) -- cgit v1.2.3-2-g168b From 12aab28511d12c58864b0077262957985fa7f2a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Thu, 15 Aug 2013 12:00:21 -0300 Subject: Verbose mode is optional MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolás Reynolds --- db-sync | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/db-sync b/db-sync index d86d348..81dee24 100755 --- a/db-sync +++ b/db-sync @@ -12,15 +12,18 @@ # * Sync repo => repo # TODO -# * verbose mode # * make a tarball of files used for forensics # * get files db +# Run as `V=true db-sync` to get verbose output +VERBOSE=${V} +${VERBOSE} && extra="-v" + # Returns contents of a repo get_repos() { mkdir -p ${TMPDIR}/$0.$$.cache # Exclude everything but db files - rsync -vmrtlH --no-p --include="*/" \ + rsync ${extra} -mrtlH --no-p --include="*/" \ --include="*.db" \ --include="*${DBEXT}" \ --exclude="*" \ @@ -93,7 +96,7 @@ init() { # Sync excluding everything but whitelist # We delete here for cleanup - rsync -vrtlH \ + rsync ${extra} -rtlH \ --delete-after \ --delete-excluded \ --delay-updates \ @@ -106,7 +109,7 @@ init() { whitelists+=(/tmp/${_repo}-${_arch}.whitelist) msg "Putting databases back in place" - rsync -vrtlH \ + rsync ${extra} -rtlH \ --delay-updates \ --safe-links \ ${TMPDIR}/$0.$$.cache/${_repo}/os/${_arch}/ \ @@ -128,7 +131,7 @@ init() { # *Don't delete-after*, this is the job of cleanup scripts. It will remove our # packages too for PKGPOOL in ${PKGPOOLS[@]}; do - rsync -vrtlH \ + rsync ${extra} -rtlH \ --delay-updates \ --safe-links \ --include-from=/tmp/any.whitelist \ @@ -146,7 +149,7 @@ init() { # *Don't delete-after*, this is the job of cleanup scripts. It will remove our # packages too for SRCPOOL in ${SRCPOOLS[@]}; do - rsync -vrtlH \ + rsync ${extra} -rtlH \ --delay-updates \ --safe-links \ --include-from=/tmp/any.whitelist \ -- cgit v1.2.3-2-g168b From e0a2005c59d15ef462d2bf26404a5b2d14266f0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Fri, 16 Aug 2013 18:50:49 -0300 Subject: add coherence repo --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 7601527..42842de 100644 --- a/config +++ b/config @@ -10,7 +10,7 @@ OURREPOS=('libre' 'libre-testing') # User repos USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde') # Community project repos -PROJREPOS=('pcr' 'social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross' 'java' 'java-ugly') +PROJREPOS=('pcr' 'social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross' 'java' 'java-ugly' 'coherence') # Remote repos RMTREPOS=('connos' 'connos-extra') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) -- cgit v1.2.3-2-g168b From 21a1c0b1c74e877209a0a467d2b9fade996cac92 Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 19 Aug 2013 16:04:18 -0700 Subject: get-repos: parabolaweb wants *.files.tar.gz files, not *.db.tar.gz --- get-repos | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/get-repos b/get-repos index 4397d54..98a39bc 100755 --- a/get-repos +++ b/get-repos @@ -26,7 +26,7 @@ DBLIST=() # Repos for _repo in ${PKGREPOS[@]}; do for _arch in ${ARCHES[@]}; do - DBLIST+=("http://repo.parabolagnulinux.org/${_repo}/os/${_arch}/${_repo}${DBEXT}") + DBLIST+=("http://repo.parabolagnulinux.org/${_repo}/os/${_arch}/${_repo}${FILESEXT}") done done @@ -36,7 +36,7 @@ done # MAYBE run scripts on get-repos.d/ which should return db urls for _repo in ${RMTREPOS}; do _arch=i586 - DBLIST+=("http://www.connochaetos.org/os/${_arch}/${_repo}/${_repo}${DBEXT}") + DBLIST+=("http://www.connochaetos.org/os/${_arch}/${_repo}/${_repo}${FILESEXT}") done # Get them all @@ -52,7 +52,7 @@ wget --directory-prefix=${TMPDIR} \ arch_re="$(echo "(${ARCHES[@]} i586)" | tr ' ' '|')" msg "Adding to parabolaweb" -find "${TMPDIR}" -iname "*${DBEXT}" | while read _db; do +find "${TMPDIR}" -iname "*${FILESEXT}" | while read _db; do _arch=$(echo "${_db}" | egrep -o "${arch_re}") if [ -z "${_arch}" ]; then -- cgit v1.2.3-2-g168b From 2bb8c19bdff6b45dbb2ce09d7f45ca42d56bcd4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Alexandre=20Silva=20Delgado?= Date: Tue, 20 Aug 2013 00:10:01 -0300 Subject: adding nonprism repo --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 42842de..5bff50a 100644 --- a/config +++ b/config @@ -10,7 +10,7 @@ OURREPOS=('libre' 'libre-testing') # User repos USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde') # Community project repos -PROJREPOS=('pcr' 'social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross' 'java' 'java-ugly' 'coherence') +PROJREPOS=('pcr' 'social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross' 'java' 'java-ugly' 'coherence' 'nonprism') # Remote repos RMTREPOS=('connos' 'connos-extra') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) -- cgit v1.2.3-2-g168b From f8051126147e57292eeaa730f2c499aea1b1a16f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Alexandre=20Silva=20Delgado?= Date: Tue, 20 Aug 2013 00:20:13 -0300 Subject: fixing PKGEXT variable --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 5bff50a..0eb851a 100644 --- a/config +++ b/config @@ -45,7 +45,7 @@ OURARCHES=(mips64el) ARCHES=(${ARCHARCHES[@]} ${OURARCHES[@]}) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" -PKGEXT=".pkg.tar.?z" +PKGEXT=".pkg.tar.xz" SRCEXT=".src.tar.gz" MAKEPKGCONF="~/.makepkg.conf" -- cgit v1.2.3-2-g168b From 48995748c99dee29ca473463ffbf8b443b6c287a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Alexandre=20Silva=20Delgado?= Date: Tue, 20 Aug 2013 00:28:04 -0300 Subject: fixing db-move for any package architecture --- db-move | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/db-move b/db-move index 010d941..73c4517 100755 --- a/db-move +++ b/db-move @@ -47,7 +47,9 @@ for pkgbase in ${args[@]:2}; do for pkgname in ${pkgnames[@]}; do for tarch in ${tarches[@]}; do - getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT} >/dev/null + getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT} >/dev/null \ + || \ + getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-any${PKGEXT} >/dev/null done done continue 2 -- cgit v1.2.3-2-g168b From e95d49c335723a15031083287d889c56a531ef1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Alexandre=20Silva=20Delgado?= Date: Tue, 20 Aug 2013 00:34:28 -0300 Subject: revert back db-move --- db-move | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/db-move b/db-move index 73c4517..010d941 100755 --- a/db-move +++ b/db-move @@ -47,9 +47,7 @@ for pkgbase in ${args[@]:2}; do for pkgname in ${pkgnames[@]}; do for tarch in ${tarches[@]}; do - getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT} >/dev/null \ - || \ - getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-any${PKGEXT} >/dev/null + getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT} >/dev/null done done continue 2 -- cgit v1.2.3-2-g168b From 2a4a1c4015a4cc48f1613b7fd360148378775445 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Alexandre=20Silva=20Delgado?= Date: Wed, 21 Aug 2013 17:48:40 -0300 Subject: rename unfree to nonfree --- db-check-nonfree | 2 +- yf/PKGBUILD | 2 +- yf/your-freedom.install | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/db-check-nonfree b/db-check-nonfree index ab6491d..ecad3b9 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -33,7 +33,7 @@ for repo in ${ARCHREPOS[@]}; do fi done if [ ${#cleanpkgs[@]} -ge 1 ]; then - msg2 "Unfree: ${cleanpkgs[@]}" + msg2 "Nonfree: ${cleanpkgs[@]}" arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]} fi done diff --git a/yf/PKGBUILD b/yf/PKGBUILD index cc6f07b..e737482 100644 --- a/yf/PKGBUILD +++ b/yf/PKGBUILD @@ -2,7 +2,7 @@ pkgname=your-freedom pkgver=$(LC_ALL=C date -u +%Y%m%d) pkgrel=1 -pkgdesc="This package conflicts with every unfree package known to date." +pkgdesc="This package conflicts with every nonfree package known to date." arch=('any') url="https://parabolagnulinux.org" license=('GPL') diff --git a/yf/your-freedom.install b/yf/your-freedom.install index 49ae045..731a575 100644 --- a/yf/your-freedom.install +++ b/yf/your-freedom.install @@ -3,18 +3,18 @@ pre_install() { cat < Date: Thu, 22 Aug 2013 12:54:10 -0300 Subject: remove coherence repo --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 0eb851a..2c1fa43 100644 --- a/config +++ b/config @@ -10,7 +10,7 @@ OURREPOS=('libre' 'libre-testing') # User repos USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde') # Community project repos -PROJREPOS=('pcr' 'social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross' 'java' 'java-ugly' 'coherence' 'nonprism') +PROJREPOS=('pcr' 'social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross' 'java' 'java-ugly' 'nonprism') # Remote repos RMTREPOS=('connos' 'connos-extra') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) -- cgit v1.2.3-2-g168b From 3e085d0d529a749386f826ecb2dd189df55b4833 Mon Sep 17 00:00:00 2001 From: Icarious Date: Mon, 26 Aug 2013 05:26:11 +0530 Subject: removed unnecessary code --- cron-jobs/sourceballs2 | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index 5e228fc..5644268 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -60,32 +60,3 @@ for repo in ${PKGREPOS[@]}; do popd >/dev/null done # end repos - -# Cleanup old source packages -cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" -cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort" -old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) - -if [ ${#old_pkgs[@]} -ge 1 ]; then - msg "Removing old source packages..." - ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' - for old_pkg in ${old_pkgs[@]}; do - msg2 "${old_pkg}" - if ! ${SOURCE_CLEANUP_DRYRUN}; then - mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" - touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" - fi - done -fi - -old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -name "*${SRCEXT}" -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) -if [ ${#old_pkgs[@]} -ge 1 ]; then - msg "Removing old source packages from the cleanup directory..." - for old_pkg in ${old_pkgs[@]}; do - msg2 "${old_pkg}" - ${SOURCE_CLEANUP_DRYRUN} || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" - done -fi - -script_unlock - -- cgit v1.2.3-2-g168b From f185efe31d518b08b45e07288a4239eddaac11b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Mon, 26 Aug 2013 16:36:21 -0300 Subject: remove deprecated repos --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 2c1fa43..f390bea 100644 --- a/config +++ b/config @@ -10,7 +10,7 @@ OURREPOS=('libre' 'libre-testing') # User repos USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde') # Community project repos -PROJREPOS=('pcr' 'social' 'gis' 'artistic' 'elementary' 'kernels' 'radio' 'security' 'sugar' 'gnu' 'cross' 'java' 'java-ugly' 'nonprism') +PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'java-ugly' 'nonprism') # Remote repos RMTREPOS=('connos' 'connos-extra') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) -- cgit v1.2.3-2-g168b From 97ed9a26bd00c8a4c03a331c076e9f2a9c7ac57f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Mon, 26 Aug 2013 16:47:10 -0300 Subject: remove fauno repo --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index f390bea..0882c86 100644 --- a/config +++ b/config @@ -8,7 +8,7 @@ ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos -USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde') +USERREPOS=('~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde') # Community project repos PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'java-ugly' 'nonprism') # Remote repos -- cgit v1.2.3-2-g168b From c2513af7d1a4ab9180cbcfd34135cd04add73c0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Wed, 28 Aug 2013 18:18:53 -0300 Subject: remove mtjm repo --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 0882c86..6c652a7 100644 --- a/config +++ b/config @@ -8,7 +8,7 @@ ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos -USERREPOS=('~smv' '~xihh' '~mtjm' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde') +USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde') # Community project repos PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'java-ugly' 'nonprism') # Remote repos -- cgit v1.2.3-2-g168b From 93229619764ff3c59fd097c72fb7890a7f542668 Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 14 Oct 2013 22:32:27 +0000 Subject: config: only set staging if it is not set from the environment --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 6c652a7..03a80bd 100644 --- a/config +++ b/config @@ -38,7 +38,7 @@ REQUIRE_SIGNATURE=true LOCK_DELAY=10 LOCK_TIMEOUT=300 -STAGING="$FTP_BASE/staging" +[ -n "${STAGING:-}" ] || STAGING="$FTP_BASE/staging" TMPDIR="/tmp" ARCHARCHES=(i686 x86_64) OURARCHES=(mips64el) -- cgit v1.2.3-2-g168b From ce88f47d19cb11ebcc02565156deddc6b48df38c Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 14 Oct 2013 18:41:54 -0400 Subject: db-sync: also generate ${repo}.files databases for imported repos --- db-sync | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/db-sync b/db-sync index 81dee24..684f7e6 100755 --- a/db-sync +++ b/db-sync @@ -26,6 +26,8 @@ get_repos() { rsync ${extra} -mrtlH --no-p --include="*/" \ --include="*.db" \ --include="*${DBEXT}" \ + --include="*.db" \ + --include="*${FILESEXT}" \ --exclude="*" \ --delete-after \ rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/$0.$$.cache @@ -46,9 +48,7 @@ get_blacklist() { # repo # arch get_repo_file() { -# shopt -s nullglob - - echo "${TMPDIR}/$0.$$.cache/${1}/os/${2}/${1}${DBEXT}" + echo "${TMPDIR}/$0.$$.cache/${1}/os/${2}/${1}" } # Process the databases and get the libre packages @@ -69,21 +69,27 @@ init() { for _arch in ${ARCHARCHES[@]}; do msg "Processing ${_repo}-${_arch}" - repo_file=$(get_repo_file ${_repo} ${_arch}) + db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT} + files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT} - if [ ! -f "${repo_file}" ]; then - warning "${repo_file} doesn't exist, skipping this repo-arch" + if [ ! -f "${db_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${db_file}" + continue + fi + if [ ! -f "${files_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${files_file}" continue fi # Remove blacklisted packages and count them # TODO capture all removed packages for printing on debug mode - msg2 "Removing blacklisted packages: $( - LC_ALL=C repo-remove ${repo_file} ${blacklist[@]} 2>&1 | \ - grep "\-> Removing" 2>/dev/null| wc -l)" + msg2 "Removing blacklisted packages from .db database..." + LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" + msg2 "Removing blacklisted packages from .files database..." + LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" # Get db contents - db=($(get_repo_content ${repo_file})) + db=($(get_repo_content ${db_file})) msg2 "Process clean db for syncing..." -- cgit v1.2.3-2-g168b From 45bfa89440e93cd722e1c14dcd58b60a52808fa4 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 14 Oct 2013 18:43:20 -0400 Subject: oops --- db-sync | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-sync b/db-sync index 684f7e6..e8481d6 100755 --- a/db-sync +++ b/db-sync @@ -26,7 +26,7 @@ get_repos() { rsync ${extra} -mrtlH --no-p --include="*/" \ --include="*.db" \ --include="*${DBEXT}" \ - --include="*.db" \ + --include="*.files" \ --include="*${FILESEXT}" \ --exclude="*" \ --delete-after \ -- cgit v1.2.3-2-g168b From 280421c67b7042ea94fc2ec94f04278cb72e2a06 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 30 Nov 2013 03:17:18 +0000 Subject: Change the default WEB_DIR to reflect the new server --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 03a80bd..3d15c1b 100644 --- a/config +++ b/config @@ -52,4 +52,4 @@ MAKEPKGCONF="~/.makepkg.conf" BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" # parabolaweb root -WEB_DIR=/srv/http/web +WEB_DIR=/srv/http/parabolagnulinux.org/web -- cgit v1.2.3-2-g168b From 8a693d7f3a9324de445b5a151c1f46194405f0eb Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 30 Nov 2013 03:17:57 +0000 Subject: Drop remote (connos) repo support --- config | 1 - get-repos | 9 --------- 2 files changed, 10 deletions(-) diff --git a/config b/config index 3d15c1b..e323d00 100644 --- a/config +++ b/config @@ -12,7 +12,6 @@ USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jor # Community project repos PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'java-ugly' 'nonprism') # Remote repos -RMTREPOS=('connos' 'connos-extra') PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) PKGPOOL='pool/packages' SRCPOOL='sources/packages' diff --git a/get-repos b/get-repos index 98a39bc..bfc08ff 100755 --- a/get-repos +++ b/get-repos @@ -30,15 +30,6 @@ for _repo in ${PKGREPOS[@]}; do done done -# Remote repos -# TODO don't hardcode the remote architecture -# TODO don't hardcode the remote url -# MAYBE run scripts on get-repos.d/ which should return db urls -for _repo in ${RMTREPOS}; do - _arch=i586 - DBLIST+=("http://www.connochaetos.org/os/${_arch}/${_repo}/${_repo}${FILESEXT}") -done - # Get them all msg "Retrieving ${#DBLIST[@]} databases" wget --directory-prefix=${TMPDIR} \ -- cgit v1.2.3-2-g168b From a88674528c5df02f03bf30449982932b6cc378dc Mon Sep 17 00:00:00 2001 From: Drtan Samos Date: Tue, 17 Dec 2013 10:24:33 +0100 Subject: Added a user repository for me in order to test packages before they come to the offical repositories. --- config | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config b/config index e323d00..9baf816 100644 --- a/config +++ b/config @@ -8,7 +8,8 @@ ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos -USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde') +USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' +'~coadde' '~drtan') # Community project repos PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'java-ugly' 'nonprism') # Remote repos -- cgit v1.2.3-2-g168b From ea21f1388681d4fed43201bda54efc9d63aadc88 Mon Sep 17 00:00:00 2001 From: Drtan Samos Date: Tue, 17 Dec 2013 10:36:31 +0100 Subject: Removes the extra new line added by nano for unknown reasons. --- config | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config b/config index 9baf816..8b8117f 100644 --- a/config +++ b/config @@ -8,8 +8,7 @@ ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib') # Official Parabola repos OURREPOS=('libre' 'libre-testing') # User repos -USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' -'~coadde' '~drtan') +USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan') # Community project repos PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'java-ugly' 'nonprism') # Remote repos -- cgit v1.2.3-2-g168b From 77fa9fc45569325a508893e16bcbb404fe12ffbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Fri, 27 Dec 2013 20:19:45 -0300 Subject: Original abslibre sync script used on the dead repo --- abslibre | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/abslibre b/abslibre index 9485986..e49d627 100755 --- a/abslibre +++ b/abslibre @@ -30,18 +30,21 @@ function get_blacklist() { return 1 } +# Prevent using an empty blacklist + [ $(wc -l ${BLFILE} | cut -d " " -f1) -eq 0 ] && return 1 + printf "[OK]\n" } function sync_abs_libre() { # Clone ABSLibre git repo - if [ -d /tmp/abslibre/.git ]; then - pushd /tmp/abslibre >/dev/null 2>&1 + if [ -d /var/tmp/abslibre/.git ]; then + pushd /var/tmp/abslibre >/dev/null 2>&1 git pull popd >/dev/null 2>&1 else - git clone /srv/git/repositories/abslibre.git /tmp/abslibre + git clone /srv/git/abslibre.git /var/tmp/abslibre fi # Sync from ABS and then sync from ABSLibre @@ -51,7 +54,7 @@ function sync_abs_libre() { ${ABSROOT} \ ${ABSLIBRE} \ && - for ARCH in i686 x86_64; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --exclude=.git/ /tmp/abslibre/ ${ABSLIBRE}/${ARCH}/; done) || { + for ARCH in i686 x86_64; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --exclude=.git/ /var/tmp/abslibre/ ${ABSLIBRE}/${ARCH}/; done) || { printf "[FAILED]\n" return 1 } @@ -62,7 +65,21 @@ function sync_abs_libre() { sync_pre_mips64el() { pushd /home/parabola/abslibre-pre-mips64el >/dev/null - rsync ${SYNCARGS} --exclude=.git* ${ABSLIBRE}/x86_64/ /home/parabola/abslibre-pre-mips64el/ && git add * && git commit -m "$(date)" + rsync ${SYNCARGS} \ + --exclude=.git* \ + --exclude=community-staging \ + --exclude=community-testing \ + --exclude=gnome-unstable \ + --exclude=kde-unstable \ + --exclude=multilib \ + --exclude=multilib-testing \ + --exclude=multilib-staging \ + --exclude=staging \ + --exclude=testing \ + ${ABSLIBRE}/x86_64/ \ + /home/parabola/abslibre-pre-mips64el/ && \ + git add . && \ + git commit -m "$(date)" -a } # Create .abs.tar.gz tarballs @@ -88,6 +105,9 @@ get_blacklist || exit 1 sync_abs_libre || exit 1 # This is being done at repo server now sync_pre_mips64el || exit 1 -create_tarballs || exit 1 +#create_tarballs || exit 1 + +echo "Exclusion list used" +cat ${BLFILE} exit 0 -- cgit v1.2.3-2-g168b From 187b0eb8d443d181dc533b1d1d0fce9aa38b4c0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Fri, 27 Dec 2013 20:25:11 -0300 Subject: Remove abs dependency --- abslibre | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/abslibre b/abslibre index e49d627..70e4be3 100755 --- a/abslibre +++ b/abslibre @@ -8,7 +8,13 @@ BLACKLIST='http://repo.parabolagnulinux.org/docs/blacklist.txt' SYNCARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g' BLFILE=/tmp/blacklist.txt -. /etc/abs.conf +# Variables from abs.conf +ABSROOT="/var/abs/" +# DON'T CHANGE. WE NEED IT FOR ABSLIBRE +SYNCSERVER="rsync.archlinux.org" +ARCH="i686" +MIRRORLIST="/etc/pacman.d/mirrorlist" +REPOS=(core extra community testing community-testing !staging !community-staging) # Steps # * Sync abs -- cgit v1.2.3-2-g168b From 2e756472a8c75eafc290bc8823013f3ac4335a33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Fri, 27 Dec 2013 20:26:13 -0300 Subject: Use the blacklist from git --- abslibre | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/abslibre b/abslibre index 70e4be3..fd2e900 100755 --- a/abslibre +++ b/abslibre @@ -4,7 +4,7 @@ ABSLIBRE=/var/abslibre ABSGIT=/srv/git/repositories/abslibre.git # Remote # ABSGIT=http://projects.parabolagnulinux.org/abslibre.git -BLACKLIST='http://repo.parabolagnulinux.org/docs/blacklist.txt' +BLACKLIST='https://projects.parabolagnulinux.org/blacklist.git/plain/blacklist.txt' SYNCARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g' BLFILE=/tmp/blacklist.txt -- cgit v1.2.3-2-g168b From 863af4a1c724e9ef13d5011c84f7b6ea8e940c24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Fri, 27 Dec 2013 20:33:47 -0300 Subject: Adapt to the current setup --- abslibre | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/abslibre b/abslibre index fd2e900..bb7fd12 100755 --- a/abslibre +++ b/abslibre @@ -68,8 +68,11 @@ function sync_abs_libre() { printf "[OK]\n" } +# This part is very hacky and particular to the current setup :P sync_pre_mips64el() { - pushd /home/parabola/abslibre-pre-mips64el >/dev/null + pushd /home/fauno/Repos/abslibre-pre-mips64el >/dev/null + + sudo -u fauno sh -c " rsync ${SYNCARGS} \ --exclude=.git* \ @@ -83,9 +86,11 @@ sync_pre_mips64el() { --exclude=staging \ --exclude=testing \ ${ABSLIBRE}/x86_64/ \ - /home/parabola/abslibre-pre-mips64el/ && \ + /home/fauno/Repos/abslibre-pre-mips64el/ && \ git add . && \ - git commit -m "$(date)" -a + git commit -m \"$(date)\" -a + git push origin master + " } # Create .abs.tar.gz tarballs -- cgit v1.2.3-2-g168b From ffe9ada1c43d5fd779863c4be2782f7b690a5fbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Fri, 27 Dec 2013 20:41:02 -0300 Subject: Changed to /srv --- abslibre | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/abslibre b/abslibre index bb7fd12..fa777ac 100755 --- a/abslibre +++ b/abslibre @@ -1,6 +1,6 @@ #!/bin/bash -ABSLIBRE=/var/abslibre +ABSLIBRE=/srv/abslibre ABSGIT=/srv/git/repositories/abslibre.git # Remote # ABSGIT=http://projects.parabolagnulinux.org/abslibre.git @@ -9,7 +9,7 @@ SYNCARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g' BLFILE=/tmp/blacklist.txt # Variables from abs.conf -ABSROOT="/var/abs/" +ABSROOT="/srv/abs/" # DON'T CHANGE. WE NEED IT FOR ABSLIBRE SYNCSERVER="rsync.archlinux.org" ARCH="i686" -- cgit v1.2.3-2-g168b From f65c477e42f46a3e22b5fbc5da790f9561483770 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 31 Dec 2013 14:50:37 -0500 Subject: clean up --- TODO | 2 +- config.orig | 54 --------------------------------- cron-jobs/ftpdir-cleanup | 7 +++++ cron-jobs/sourceballs | 8 ++--- db-functions | 19 +++++------- db-update | 9 +++--- git-pbs | 44 --------------------------- libremessages | 77 ------------------------------------------------ 8 files changed, 23 insertions(+), 197 deletions(-) delete mode 100644 config.orig delete mode 100755 git-pbs delete mode 100755 libremessages diff --git a/TODO b/TODO index 3219b1c..9dd4b52 100644 --- a/TODO +++ b/TODO @@ -7,4 +7,4 @@ * Fix db-move - - Make it use abslibre \ No newline at end of file + - Make it use abslibre diff --git a/config.orig b/config.orig deleted file mode 100644 index a32f82f..0000000 --- a/config.orig +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -<<<<<<< HEAD -FTP_BASE="/srv/http/repo/public" -ARCH_BASE="/srv/http/repo/public" -SVNREPO="/srv/http/repo/abslibre" -======= -FTP_BASE="/srv/http/repo/public/temprepo" -ARCH_BASE="/srv/http/repo/public/temprepo" -SVNREPO="/var/abs" ->>>>>>> 801ea2c927ace5ee892209dd8e3c1044e1b3842e - -# Repos from Arch -ARCHREPOS=('core' 'testing') #'extra' 'community' 'testing' 'multilib') -# Official Parabola repos -OURREPOS=('libre' 'libre-testing') -# User repos -USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan') -# Community project repos -PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'sugar') -PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) -PKGPOOL='pool/packages' -SRCPOOL='sources/packages' - -CLEANUP_DESTDIR="$FTP_BASE/old/packages" -CLEANUP_DRYRUN=true -# Time in days to keep moved packages -CLEANUP_KEEP=30 - -SOURCE_CLEANUP_DESTDIR="$FTP_BASE/old/sources" -SOURCE_CLEANUP_DRYRUN=true -# Time in days to keep moved sourcepackages -SOURCE_CLEANUP_KEEP=30 - -REQUIRE_SIGNATURE=true - -LOCK_DELAY=10 -LOCK_TIMEOUT=300 - -STAGING="$FTP_BASE/staging" -TMPDIR="/tmp" -ARCHARCHES=(i686 x86_64) -OURARCHES=(mips64el) -ARCHES=(${ARCHARCHES[@]} ${OURARCHES[@]}) -DBEXT=".db.tar.gz" -FILESEXT=".files.tar.gz" -PKGEXT=".pkg.tar.?z" -SRCEXT=".src.tar.gz" - -<<<<<<< HEAD -MAKEPKGCONF="~/.makepkg.conf" -======= -MAKEPKGCONF="/etc/makepkg.conf" ->>>>>>> 801ea2c927ace5ee892209dd8e3c1044e1b3842e -BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index a2823d4..e42a1a8 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -22,6 +22,13 @@ clean_pkg() { fi } +script_lock + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_lock ${repo} ${arch} || exit 1 + done +done ${CLEANUP_DRYRUN} && warning 'dry run mode is active' diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 91bc3d6..1542499 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -21,10 +21,10 @@ renice +10 -p $$ > /dev/null for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do # Repo does not exist; skip it - if [ ! -f "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue fi - bsdtar -xOf "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ + bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ | awk '/^%NAME%/ { getline b }; /^%BASE%/ { getline b }; /^%VERSION%/ { getline v }; @@ -46,7 +46,7 @@ for repo in ${PKGREPOS[@]}; do done # Create a list of all available source package file names -find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" +find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" # Check for all packages if we need to build a source package for repo in ${PKGREPOS[@]}; do @@ -59,7 +59,7 @@ for repo in ${PKGREPOS[@]}; do pkgarch=${pkginfo[2]} pkglicense=(${pkginfo[@]:3}) - # Should this packages be skipped? + # Should this package be skipped? if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then continue fi diff --git a/db-functions b/db-functions index 072f43d..0b59a53 100644 --- a/db-functions +++ b/db-functions @@ -135,7 +135,7 @@ repo_lock () { _count=0 while [ $_count -le $_trial ] || $_lockblock ; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - _owner="$(stat -c %U $LOCKDIR)" + _owner="$(/usr/bin/stat -c %U $LOCKDIR)" warning "Repo [${1}] (${2}) is already locked by $_owner. " msg2 "Retrying in $LOCK_DELAY seconds..." else @@ -250,9 +250,6 @@ getpkgfile() { } getpkgfiles() { -# Ignore anything that doesn't glob to PKGEXT - shopt -s nullglob - local f if [ ! -z "$(echo ${@%\.*} | sed "s/ /\n/g" | sort | uniq -D)" ]; then error 'Duplicate packages found!' @@ -270,8 +267,6 @@ getpkgfiles() { done echo ${@} - - shopt -u nullglob } check_pkgfile() { @@ -330,7 +325,7 @@ check_splitpkgs() { for pkgfile in ${pkgfiles[@]}; do issplitpkg "${pkgfile}" || continue local _pkgbase="$(getpkgbase ${pkgfile})" - msg2 "Checking $_pkgbase" + msg2 "Checking %s" "$_pkgbase" local _pkgname="$(getpkgname ${pkgfile})" local _pkgarch="$(getpkgarch ${pkgfile})" mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" @@ -411,7 +406,7 @@ set_repo_permission() { local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" if [ -w "${dbfile}" ]; then - local group=$(stat --printf='%G' "$(dirname "${dbfile}")") + local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") chgrp $group "${dbfile}" || error "Could not change group of ${dbfile} to $group" chgrp $group "${filesfile}" || error "Could not change group of ${filesfile} to $group" chmod g+w "${dbfile}" || error "Could not set write permission for group $group to ${dbfile}" @@ -428,9 +423,9 @@ arch_repo_add() { # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null - repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ + /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" - repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ + /usr/bin/repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ || error "repo-add -f ${repo}${FILESEXT} ${pkgs[@]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" @@ -449,9 +444,9 @@ arch_repo_remove() { error "No database found at '${dbfile}'" return 1 fi - repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ + /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ || error "repo-remove ${dbfile} ${pkgs[@]}" - repo-remove -q "${filesfile}" ${pkgs[@]} \ + /usr/bin/repo-remove -q "${filesfile}" ${pkgs[@]} \ || error "repo-remove ${filesfile} ${pkgs[@]}" set_repo_permission "${repo}" "${arch}" diff --git a/db-update b/db-update index 74d3f45..ca18ca3 100755 --- a/db-update +++ b/db-update @@ -52,10 +52,9 @@ for repo in ${repos[@]}; do die "Package ${repo}/${pkg##*/} already exists in another repository" fi done - # This is fucking obnoxious -# if ! check_splitpkgs ${repo} ${pkgs[@]}; then -# die "Missing split packages for ${repo}" -# fi + if ! check_splitpkgs ${repo} ${pkgs[@]}; then + die "Missing split packages for ${repo}" + fi else die "Could not read ${STAGING}" fi @@ -82,7 +81,7 @@ for repo in ${repos[@]}; do if [ -f "$FTP_BASE/${PKGPOOL}/${pkgfile}.sig" ]; then ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "$FTP_BASE/$repo/os/${pkgarch}" fi - add_pkgs[${#add_pkgs[*]}]=${pkgfile} + add_pkgs[${#add_pkgs[*]}]=${pkgfile} done if [ ${#add_pkgs[@]} -ge 1 ]; then arch_repo_add "${repo}" "${pkgarch}" ${add_pkgs[@]} diff --git a/git-pbs b/git-pbs deleted file mode 100755 index b815863..0000000 --- a/git-pbs +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -_pkg=$1 - -mkdir -p $_pkg -pushd $_pkg - - -if [ ! -d .git ]; then -# Start a git repo for the package -# Add the remote origin -# Pull the package branch onto an unmodified branch - git init - git remote add arch git://projects.archlinux.org/svntogit/packages.git - -# Export the repository - touch .git/git-daemon-export-ok - -# Pass the -b flag to checkout to create the branches - extra="-b" -fi - -git checkout ${extra} upstream -git pull arch packages/$_pkg - -# Move PKGBUILD and files to the basedir -# Remove everything else from the repo -git checkout ${extra} master - -# This produces a lot of merging conflicts -git merge upstream - -# This apparently solves them -git mv trunk/* . -git rm -rf repos - -# Remove the actual files -rm -rf trunk repos - -# Commit everything -git commit -a -m "Converted to PBS" - -# Return to the repo -popd >/dev/null diff --git a/libremessages b/libremessages deleted file mode 100755 index 9fbbc2b..0000000 --- a/libremessages +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) 2006-2010 Pacman Development Team -# Copyright (c) 2002-2006 by Judd Vinet -# Copyright (c) 2005 by Aurelien Foret -# Copyright (c) 2006 by Miklos Vajna -# Copyright (c) 2005 by Christian Hamar -# Copyright (c) 2006 by Alex Smith -# Copyright (c) 2006 by Andras Voroskoi -# Copyright (c) 2011 by Joshua Haase -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# gettext initialization -export TEXTDOMAIN='libretools' -export TEXTDOMAINDIR='/usr/share/locale' - -# check if messages are to be printed using color -unset ALL_OFF BOLD BLUE GREEN RED YELLOW - -if tput setaf 0 &>/dev/null; then - ALL_OFF="$(tput sgr0)" - BOLD="$(tput bold)" - BLUE="${BOLD}$(tput setaf 4)" - GREEN="${BOLD}$(tput setaf 2)" - RED="${BOLD}$(tput setaf 1)" - YELLOW="${BOLD}$(tput setaf 3)" - PURPLE="${ALL_OFF}$(tput setaf 5)" -else - ALL_OFF="\033[1;0m" - BOLD="\033[1;1m" - BLUE="${BOLD}\033[1;34m" - GREEN="${BOLD}\033[1;32m" - RED="${BOLD}\033[1;31m" - YELLOW="${BOLD}\033[1;33m" - PURPLE="${BOLD}\033[1;30;40m" -fi - -stdnull() { - local action=$1; - eval "${action} >/dev/null 2>&1" -} - -plain() { - local mesg=$1; shift - printf "${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -msg() { - local mesg=$1; shift - printf "${GREEN}==>${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -msg2() { - local mesg=$1; shift - printf "${BLUE} ->${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -warning() { - local mesg=$1; shift - printf "${YELLOW}==> $(gettext "WARNING:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -error() { - local mesg=$1; shift - printf "${RED}==> $(gettext "ERROR:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -- cgit v1.2.3-2-g168b From e210b136790f740a6eb42913b30f6d0c8557c81d Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 31 Dec 2013 14:53:28 -0500 Subject: get rid of $ARCH_BASE --- config | 1 - createrepos | 4 ++-- cron-jobs/sourceballs2 | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/config b/config index 496b3ef..3b7c13a 100644 --- a/config +++ b/config @@ -1,6 +1,5 @@ #/bin/bash # as a hint to text editors FTP_BASE="/srv/http/repo/public" -ARCH_BASE="/srv/http/repo/public" SVNREPO="/var/abs" # Repos from Arch diff --git a/createrepos b/createrepos index 4ee057b..eec26e0 100755 --- a/createrepos +++ b/createrepos @@ -3,6 +3,6 @@ source $(dirname $0)/config -mkdir -p ${FTP_BASE}/{${PKGPOOL},${SRCPOOL}} ${ARCH_BASE} ${CLEANUP_DESTDIR} ${SOURCE_CLEANUP_DESTDIR} ${STAGING} +mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR} ${STAGING}" -$(dirname $0)/create-repo ${PKGREPOS[@]} +$(dirname $0)/create-repo "${PKGREPOS[@]}" diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index 5644268..5b14d07 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -17,7 +17,7 @@ script_lock renice +10 -p $$ > /dev/null # Create a list of all available source package file names -find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" +find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" pushd "${SVNREPO}" >/dev/null -- cgit v1.2.3-2-g168b From 7a49d26565b1bdbd73d946e3dca3dae7b1073572 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 31 Dec 2013 15:30:23 -0500 Subject: db-sync: use tab indent --- db-sync | 258 ++++++++++++++++++++++++++++++++-------------------------------- 1 file changed, 129 insertions(+), 129 deletions(-) diff --git a/db-sync b/db-sync index e8481d6..39e72b9 100755 --- a/db-sync +++ b/db-sync @@ -22,157 +22,157 @@ ${VERBOSE} && extra="-v" # Returns contents of a repo get_repos() { mkdir -p ${TMPDIR}/$0.$$.cache -# Exclude everything but db files - rsync ${extra} -mrtlH --no-p --include="*/" \ - --include="*.db" \ - --include="*${DBEXT}" \ - --include="*.files" \ - --include="*${FILESEXT}" \ - --exclude="*" \ - --delete-after \ - rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/$0.$$.cache + # Exclude everything but db files + rsync ${extra} -mrtlH --no-p --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --include="*.files" \ + --include="*${FILESEXT}" \ + --exclude="*" \ + --delete-after \ + rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/$0.$$.cache } get_repo_content() { -# Return all contents - bsdtar tf ${1} | \ - cut -d "/" -f 1 | \ - sort -u + # Return all contents + bsdtar tf ${1} | \ + cut -d "/" -f 1 | \ + sort -u } # Prints blacklisted packages get_blacklist() { - cut -d ':' -f 1 "${BLACKLIST_FILE}" + cut -d ':' -f 1 "${BLACKLIST_FILE}" } # repo # arch get_repo_file() { - echo "${TMPDIR}/$0.$$.cache/${1}/os/${2}/${1}" + echo "${TMPDIR}/$0.$$.cache/${1}/os/${2}/${1}" } # Process the databases and get the libre packages init() { -# Get the blacklisted packages - blacklist=($(get_blacklist)) -# Store all the whitelist files - whitelists=() - - msg "${#blacklist[@]} packages in blacklist" - -# Sync the repos databases - get_repos - -# Traverse all repo-arch pairs - for _repo in ${ARCHREPOS[@]}; do - for _arch in ${ARCHARCHES[@]}; do - msg "Processing ${_repo}-${_arch}" - - db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT} - files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT} - - if [ ! -f "${db_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${db_file}" - continue - fi - if [ ! -f "${files_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${files_file}" - continue - fi - -# Remove blacklisted packages and count them -# TODO capture all removed packages for printing on debug mode - msg2 "Removing blacklisted packages from .db database..." - LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" - msg2 "Removing blacklisted packages from .files database..." - LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" - -# Get db contents - db=($(get_repo_content ${db_file})) - - msg2 "Process clean db for syncing..." - -# Create a whitelist, add * wildcard to end -# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if -# we aren't syncing them - echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist - - msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" - -# Sync excluding everything but whitelist -# We delete here for cleanup - rsync ${extra} -rtlH \ - --delete-after \ - --delete-excluded \ - --delay-updates \ - --include-from=/tmp/${_repo}-${_arch}.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ - -# Add a new whitelist - whitelists+=(/tmp/${_repo}-${_arch}.whitelist) - - msg "Putting databases back in place" - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - ${TMPDIR}/$0.$$.cache/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ - -# Cleanup - unset db - done - done - - - msg "Syncing package pool" -# Concatenate all whitelists - cat ${whitelists[@]} | sort -u > /tmp/any.whitelist - - msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" - -# Sync -# *Don't delete-after*, this is the job of cleanup scripts. It will remove our -# packages too - for PKGPOOL in ${PKGPOOLS[@]}; do - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ - ${FTP_BASE}/${PKGPOOL}/ - done - -# Sync sources - msg "Syncing source pool" - #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist - - #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool" -# Sync -# *Don't delete-after*, this is the job of cleanup scripts. It will remove our -# packages too - for SRCPOOL in ${SRCPOOLS[@]}; do - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ - ${FTP_BASE}/${SRCPOOL}/ - done + # Get the blacklisted packages + blacklist=($(get_blacklist)) + # Store all the whitelist files + whitelists=() + + msg "${#blacklist[@]} packages in blacklist" + + # Sync the repos databases + get_repos + + # Traverse all repo-arch pairs + for _repo in ${ARCHREPOS[@]}; do + for _arch in ${ARCHARCHES[@]}; do + msg "Processing ${_repo}-${_arch}" + + db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT} + files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT} + + if [ ! -f "${db_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${db_file}" + continue + fi + if [ ! -f "${files_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${files_file}" + continue + fi + + # Remove blacklisted packages and count them + # TODO capture all removed packages for printing on debug mode + msg2 "Removing blacklisted packages from .db database..." + LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" + msg2 "Removing blacklisted packages from .files database..." + LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" + + # Get db contents + db=($(get_repo_content ${db_file})) + + msg2 "Process clean db for syncing..." + + # Create a whitelist, add * wildcard to end + # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if + # we aren't syncing them + echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist + + msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" + + # Sync excluding everything but whitelist + # We delete here for cleanup + rsync ${extra} -rtlH \ + --delete-after \ + --delete-excluded \ + --delay-updates \ + --include-from=/tmp/${_repo}-${_arch}.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + + # Add a new whitelist + whitelists+=(/tmp/${_repo}-${_arch}.whitelist) + + msg "Putting databases back in place" + rsync ${extra} -rtlH \ + --delay-updates \ + --safe-links \ + ${TMPDIR}/$0.$$.cache/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + + # Cleanup + unset db + done + done + + + msg "Syncing package pool" + # Concatenate all whitelists + cat ${whitelists[@]} | sort -u > /tmp/any.whitelist + + msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" + + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + for PKGPOOL in ${PKGPOOLS[@]}; do + rsync ${extra} -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ + ${FTP_BASE}/${PKGPOOL}/ + done + + # Sync sources + msg "Syncing source pool" + #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist + + #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool" + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + for SRCPOOL in ${SRCPOOLS[@]}; do + rsync ${extra} -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ + ${FTP_BASE}/${SRCPOOL}/ + done -# Cleanup - unset blacklist whitelists _arch _repo repo_file + # Cleanup + unset blacklist whitelists _arch _repo repo_file } trap_exit() { - echo - error "$@" - exit 1 + echo + error "$@" + exit 1 } -- cgit v1.2.3-2-g168b From 9b69d5863bdf398b78ce46661dabb9abec09385e Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 31 Dec 2013 15:31:33 -0500 Subject: db-sync: use mktemp/trap --- db-sync | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/db-sync b/db-sync index 39e72b9..0acfbc1 100755 --- a/db-sync +++ b/db-sync @@ -19,9 +19,11 @@ VERBOSE=${V} ${VERBOSE} && extra="-v" +WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") +trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT + # Returns contents of a repo get_repos() { - mkdir -p ${TMPDIR}/$0.$$.cache # Exclude everything but db files rsync ${extra} -mrtlH --no-p --include="*/" \ --include="*.db" \ @@ -30,7 +32,7 @@ get_repos() { --include="*${FILESEXT}" \ --exclude="*" \ --delete-after \ - rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/$0.$$.cache + rsync://${mirror}/${mirrorpath}/ "$WORKDIR" } get_repo_content() { @@ -48,7 +50,7 @@ get_blacklist() { # repo # arch get_repo_file() { - echo "${TMPDIR}/$0.$$.cache/${1}/os/${2}/${1}" + echo "${WORKDIR}/${1}/os/${2}/${1}" } # Process the databases and get the libre packages @@ -118,7 +120,7 @@ init() { rsync ${extra} -rtlH \ --delay-updates \ --safe-links \ - ${TMPDIR}/$0.$$.cache/${_repo}/os/${_arch}/ \ + "${WORKDIR}/${_repo}/os/${_arch}/" \ ${FTP_BASE}/${_repo}/os/${_arch}/ # Cleanup @@ -188,5 +190,3 @@ trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR init - -rm -r ${TMPDIR}/$0.$$.cache -- cgit v1.2.3-2-g168b From 6a093f1dc6bd5398115544dd229b520f9432e122 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 31 Dec 2013 15:49:06 -0500 Subject: use `readlink -e` on $0 --- any-to-ours | 6 +++--- create-repo | 6 +++--- createrepos | 4 ++-- cron-jobs/integrity-check | 2 +- cron-jobs/repo-sanity-check | 4 ++-- cron-jobs/sourceballs2 | 2 +- cron-jobs/update-abs-tarballs | 2 +- db-check-nonfree | 4 ++-- db-cleanup | 6 +++--- db-list-unsigned-packages | 6 +++--- db-sync | 6 +++--- get-repos | 6 +++--- migrate-repo | 2 +- mkrepo | 4 ++-- repo-restore-to-normal | 4 ++-- testing2x | 2 +- yf-update | 12 ++++++------ 17 files changed, 39 insertions(+), 39 deletions(-) diff --git a/any-to-ours b/any-to-ours index a1d6686..31fea5f 100755 --- a/any-to-ours +++ b/any-to-ours @@ -7,9 +7,9 @@ trap_exit() { exit 1 } -source $(dirname $0)/config -source $(dirname $0)/local_config -source $(dirname $0)/libremessages +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/local_config" +source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg set -E diff --git a/create-repo b/create-repo index 58842c3..f77fa24 100755 --- a/create-repo +++ b/create-repo @@ -1,8 +1,8 @@ #!/bin/bash # Creates repository structure -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" if [ $# -eq 0 ]; then msg "Usage: $0 repo1 [repo2 ... repoX]" @@ -21,4 +21,4 @@ for _repo in $@; do done done -msg "Don't forget to add them to the PKGREPOS array on $(dirname $0)/config" +msg "Don't forget to add them to the PKGREPOS array on %s/config" "$(dirname "$(readlink -e "$0")")" diff --git a/createrepos b/createrepos index eec26e0..cd17e4e 100755 --- a/createrepos +++ b/createrepos @@ -1,8 +1,8 @@ #!/bin/bash # Creates the repo structure defined in config -source $(dirname $0)/config +source "$(dirname "$(readlink -e "$0")")/config" mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR} ${STAGING}" -$(dirname $0)/create-repo "${PKGREPOS[@]}" +"$(dirname "$(readlink -e "$0")")/create-repo" "${PKGREPOS[@]}" diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index f6c26cf..86a8f1d 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -1,6 +1,6 @@ #!/bin/bash -dirname="$(dirname $0)" +dirname="$(dirname "$(readlink -e "$0")")" . "${dirname}/../config" . "${dirname}/../db-functions" diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index 1ba90a6..2aa7892 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -1,8 +1,8 @@ #!/bin/bash # Solves issue165 -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" +. "$(dirname "$(readlink -e "$0")")/../config" # Traverse all repos for _repo in ${PKGREPOS[@]}; do diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index 5b14d07..bbe227d 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -4,7 +4,7 @@ # Makepkg --allsource every package # Remove the old sourceballs -dirname="$(dirname $(readlink -e $0))" +dirname="$(dirname "$(readlink -e "$0")")" . "${dirname}/../db-functions" . "${dirname}/../config" . "${MAKEPKGCONF}" diff --git a/cron-jobs/update-abs-tarballs b/cron-jobs/update-abs-tarballs index 824ac34..901cc4b 100755 --- a/cron-jobs/update-abs-tarballs +++ b/cron-jobs/update-abs-tarballs @@ -1,6 +1,6 @@ #!/bin/bash -. "$(dirname $0)/../config" +. "$(dirname "$(readlink -e "$0")")/../config" rsync -av --exclude=staging/ parabolagnulinux.org::abstar/ ${FTP_BASE}/ diff --git a/db-check-nonfree b/db-check-nonfree index ecad3b9..ba5f5aa 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" if [ $# -ge 1 ]; then warning "Calling $(basename $0) with a specific repository is not supported" diff --git a/db-cleanup b/db-cleanup index 904c06e..c6e4f1c 100755 --- a/db-cleanup +++ b/db-cleanup @@ -15,9 +15,9 @@ trap_exit() { exit 1 } -source $(dirname $0)/config -source $(dirname $0)/local_config -source $(dirname $0)/libremessages +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/local_config" +source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg set -E diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages index 3b5a5bd..4e90d42 100755 --- a/db-list-unsigned-packages +++ b/db-list-unsigned-packages @@ -20,8 +20,8 @@ set -e # unsigned packages available for architecture $1 and specified for # architecture $2 (usually $1 or any, default is to list all). -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" if [ $# -lt 1 ]; then msg "usage: $(basename $0) " @@ -34,5 +34,5 @@ shift for repo in ${PKGREPOS[@]} do db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" - [ -f "$db" ] && "$(dirname $0)/db-list-unsigned-packages.py" "$repo" "$@" < "$db" + [ -f "$db" ] && "$(dirname "$(readlink -e "$0")")/db-list-unsigned-packages.py" "$repo" "$@" < "$db" done diff --git a/db-sync b/db-sync index 0acfbc1..e4b6966 100755 --- a/db-sync +++ b/db-sync @@ -178,9 +178,9 @@ trap_exit() { } -source $(dirname $0)/config -source $(dirname $0)/local_config -source $(dirname $0)/libremessages +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/local_config" +source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg set -E diff --git a/get-repos b/get-repos index bfc08ff..40d7205 100755 --- a/get-repos +++ b/get-repos @@ -9,9 +9,9 @@ trap_exit() { exit 1 } -source $(dirname $0)/config -source $(dirname $0)/local_config -source $(dirname $0)/libremessages +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/local_config" +source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg set -E diff --git a/migrate-repo b/migrate-repo index 2e44adb..751d5bd 100755 --- a/migrate-repo +++ b/migrate-repo @@ -1,6 +1,6 @@ #!/bin/bash -source $(dirname $0)/config +source "$(dirname "$(readlink -e "$0")")/config" #dryrun="--dry-run" diff --git a/mkrepo b/mkrepo index 5f704cc..10d014b 100755 --- a/mkrepo +++ b/mkrepo @@ -3,8 +3,8 @@ # License: GPLv3+ # Description: A script to quickly create new [repos] -source $(dirname $0)/config -source $(dirname $0)/local_config +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/local_config" # TODO it would be simpler to expand arrays to {element1,element2,etc} for repo in $@; do diff --git a/repo-restore-to-normal b/repo-restore-to-normal index 9463731..3636920 100755 --- a/repo-restore-to-normal +++ b/repo-restore-to-normal @@ -1,8 +1,8 @@ #!/bin/bash # Solves issue165 -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" CLEANUP_DESTDIR=/home/parabolavnx/repo/pool/restore PKGREPOS=(community) diff --git a/testing2x b/testing2x index 14610c2..6646179 100755 --- a/testing2x +++ b/testing2x @@ -52,7 +52,7 @@ for repo in ${STABLE_REPOS[@]}; do repo_unlock ${repo} ${pkgarch} done if [ -n "${pkgs[${repo}]}" ]; then - "$(dirname $0)/db-move" ${TESTING_REPO} "${repo}" ${pkgs[${repo}]} + "$(dirname "$(readlink -e "$0")")/db-move" ${TESTING_REPO} "${repo}" ${pkgs[${repo}]} fi done diff --git a/yf-update b/yf-update index 9c2131e..ee5d3eb 100755 --- a/yf-update +++ b/yf-update @@ -1,17 +1,17 @@ #!/bin/bash -source $(dirname $0)/local_config -source $(dirname $0)/config -source $(dirname $0)/libremessages +source "$(dirname "$(readlink -e "$0")")/local_config" +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/libremessages" blacklist_mtime=$(printf "%.0f" $(find ${blacklist} -printf "%T@")) -last_bl_mtime=$(cat $(dirname $0)/yftime) +last_bl_mtime=$(< "$(dirname "$(readlink -e "$0")")/yftime") if [ $blacklist_mtime -gt $last_bl_mtime ]; then - pushd $(dirname $0)/yf + pushd "$(dirname "$(readlink -e "$0")")/yf" makepkg -f find . -name "*${PKGEXT}" -exec mv {} ${STAGING}/libre \; popd - echo ${blacklist_mtime} > $(dirname $0)/yftime + echo ${blacklist_mtime} > "$(dirname "$(readlink -e "$0")")/yftime" msg2 "built and staged" else msg2 "nothing to do" -- cgit v1.2.3-2-g168b From a9d3b75c29cc82346c17787456fed009392f2029 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 31 Dec 2013 16:04:28 -0500 Subject: local_config.example: remove unused variables, update comments --- local_config.example | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/local_config.example b/local_config.example index 2280cc2..daa6874 100644 --- a/local_config.example +++ b/local_config.example @@ -1,26 +1,15 @@ -# Mirror options +#/bin/bash # as a hint to text editors +_paraboladir=~/parabolagnulinux.org + +# db-sync mirror="mirrors.eu.kernel.org" mirrorpath="mirrors/archlinux" -# Directories: they should end without / -paraboladir=~/parabolagnulinux.org -tempdir=~/tmp -archdb=${tempdir}/db -docs_dir=${paraboladir}/docs -repodir=${paraboladir}/repo -licenses_dir=${docs_dir}/pending_licenses -# End Directories - -# Files -logname=${paraboladir}/log/$(date -u +%Y%m%d-%H:%M)-repo-maintainer.log -rsout_file=${tempdir}/rsout -rsync_not_needed=${tempdir}/rsync_not_needed - -rsync_blacklist=${docs_dir}/rsyncBlacklist +# mkrepo +repodir=${_paraboladir}/repo -blacklist=${docs_dir}/blacklist.txt -whitelist=${docs_dir}/whitelist.txt +# yf-update +blacklist=${_paraboladir}/docs/blacklist.txt +whitelist=${_paraboladir}/docs/whitelist.txt -# Rsync commands -rsync_list_command="rsync -rptgoL --exclude='*.abs.tar.*' --list-only --no-motd " -rsync_update_command="rsync -rptgoL --exclude='*.abs.tar.*' --no-motd " +unset _paraboladir -- cgit v1.2.3-2-g168b From 877eaef7357c5ca171e15de16ffce055da68af2f Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 31 Dec 2013 16:06:19 -0500 Subject: remove unnecessary loadings of local_config --- any-to-ours | 1 - db-cleanup | 1 - get-repos | 1 - 3 files changed, 3 deletions(-) diff --git a/any-to-ours b/any-to-ours index 31fea5f..a1697c7 100755 --- a/any-to-ours +++ b/any-to-ours @@ -8,7 +8,6 @@ trap_exit() { } source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/local_config" source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg diff --git a/db-cleanup b/db-cleanup index c6e4f1c..364f400 100755 --- a/db-cleanup +++ b/db-cleanup @@ -16,7 +16,6 @@ trap_exit() { } source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/local_config" source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg diff --git a/get-repos b/get-repos index 40d7205..2921dd0 100755 --- a/get-repos +++ b/get-repos @@ -10,7 +10,6 @@ trap_exit() { } source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/local_config" source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg -- cgit v1.2.3-2-g168b From baf990333197d42fc25e3220b2bbc97ea03d88a4 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 31 Dec 2013 17:09:29 -0500 Subject: get-repos: clean up temp directory handling - rename $TMPDIR to $WORKDIR - respect environmental $TMPDIR - use a trap to clean up,instead of a command at the end --- get-repos | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/get-repos b/get-repos index 2921dd0..5096433 100755 --- a/get-repos +++ b/get-repos @@ -19,7 +19,9 @@ trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR -TMPDIR="$(mktemp -d /tmp/$(basename $0).XXXX)" +WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") +trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT + DBLIST=() # Repos @@ -31,7 +33,7 @@ done # Get them all msg "Retrieving ${#DBLIST[@]} databases" -wget --directory-prefix=${TMPDIR} \ +wget --directory-prefix=${WORKDIR} \ --no-verbose \ --force-directories \ --no-host-directories \ @@ -42,7 +44,7 @@ wget --directory-prefix=${TMPDIR} \ arch_re="$(echo "(${ARCHES[@]} i586)" | tr ' ' '|')" msg "Adding to parabolaweb" -find "${TMPDIR}" -iname "*${FILESEXT}" | while read _db; do +find "${WORKDIR}" -iname "*${FILESEXT}" | while read _db; do _arch=$(echo "${_db}" | egrep -o "${arch_re}") if [ -z "${_arch}" ]; then @@ -52,7 +54,3 @@ find "${TMPDIR}" -iname "*${FILESEXT}" | while read _db; do "${WEB_DIR}"/manage.py reporead "${_arch}" "${_db}" || true done - -rm -r ${TMPDIR} - -exit $? -- cgit v1.2.3-2-g168b From 12507975408257ff24f1f367b8b3b842fa779f1f Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 31 Dec 2013 17:12:16 -0500 Subject: use ${0##*/} instead of basename in "usage:" text --- create-repo | 2 +- db-check-nonfree | 2 +- db-list-unsigned-packages | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/create-repo b/create-repo index f77fa24..24b890d 100755 --- a/create-repo +++ b/create-repo @@ -5,7 +5,7 @@ . "$(dirname "$(readlink -e "$0")")/config" if [ $# -eq 0 ]; then - msg "Usage: $0 repo1 [repo2 ... repoX]" + msg "Usage: ${0##*/} repo1 [repo2 ... repoX]" exit 1 fi diff --git a/db-check-nonfree b/db-check-nonfree index ba5f5aa..661daa6 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -4,7 +4,7 @@ . "$(dirname "$(readlink -e "$0")")/config" if [ $# -ge 1 ]; then - warning "Calling $(basename $0) with a specific repository is not supported" + warning "Calling ${0##*/} with a specific repository is not supported" exit 1 fi diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages index 4e90d42..985d1c0 100755 --- a/db-list-unsigned-packages +++ b/db-list-unsigned-packages @@ -24,7 +24,7 @@ set -e . "$(dirname "$(readlink -e "$0")")/config" if [ $# -lt 1 ]; then - msg "usage: $(basename $0) " + msg "usage: ${0##*/} " exit 1 fi -- cgit v1.2.3-2-g168b From bb69db69c7d319491f67be2db77889b70aa634f0 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 1 Jan 2014 21:54:55 -0500 Subject: rm repo-remove -- in Oct 2012 fauno removed repo-add, but left this The customized repo-{add,remove} did license extraction for packages. They were based on the versions from pacman 3.5.0 Here is a diff between the stock versions from 3.5.0, and the modified version that we had: --- repo-add.sh.in 2013-12-31 18:02:13.546351038 -0500 +++ repo-remove.in 2013-12-31 18:13:19.957948677 -0500 @@ -20,6 +20,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +LICENSESDIR=/home/parabolavnx/licenses + # gettext initialization export TEXTDOMAIN='pacman' export TEXTDOMAINDIR='@localedir@' @@ -309,6 +311,22 @@ fi fi + # Extracts licenses to a common license dir + msg "Extracting license" + if bsdtar -xOf ${pkgfile} .PKGINFO | grep "license" | grep "custom" ; then + if [ -d ${LICENSESDIR}/${pkgname} ]; then + rm -r ${LICENSESDIR}/${pkgname} + fi + + # Change dir to licenses, and extract them stripping the first part of the path + bsdtar -C ${LICENSESDIR}/ --include="usr/share/licenses/" \ + --strip-components 3 -xf ${pkgfile} >/dev/null 2>&1 + + if [ $? -ne 0 ]; then + warning "This package doesn't contain a license dir" + fi + fi + return 0 } # end db_write_entry @@ -328,6 +346,12 @@ rm -rf $pkgentry pkgentry=$(find_pkgentry $pkgname) done + + msg "Removing license" + if [ -d ${LICENSESDIR}/${pkgname} ]; then + rm -r ${LICENSESDIR}/${pkgname} + fi + return $notfound } # end db_remove_entry --- repo-remove | 561 ------------------------------------------------------------ 1 file changed, 561 deletions(-) delete mode 100755 repo-remove diff --git a/repo-remove b/repo-remove deleted file mode 100755 index c4bf96f..0000000 --- a/repo-remove +++ /dev/null @@ -1,561 +0,0 @@ -#!/bin/bash -# -# repo-add - add a package to a given repo database file -# repo-remove - remove a package entry from a given repo database file -# Generated from repo-add.in; do not edit by hand. -# -# Copyright (c) 2006-2008 Aaron Griffin -# Copyright (c) 2007-2008 Dan McGee -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -LICENSESDIR=/home/parabolavnx/licenses - -# gettext initialization -export TEXTDOMAIN='pacman' -export TEXTDOMAINDIR='/usr/share/locale' - -myver='3.5.0' -confdir='/home/parabolavnx/etc' - -QUIET=0 -DELTA=0 -WITHFILES=0 -REPO_DB_FILE= -LOCKFILE= -CLEAN_LOCK=0 - -# ensure we have a sane umask set -umask 0022 - -msg() { - (( QUIET )) && return - local mesg=$1; shift - printf "==> ${mesg}\n" "$@" >&1 -} - -msg2() { - (( QUIET )) && return - local mesg=$1; shift - printf " -> ${mesg}\n" "$@" >&1 -} - -warning() { - local mesg=$1; shift - printf "==> $(gettext "WARNING:") ${mesg}\n" "$@" >&2 -} - -error() { - local mesg=$1; shift - printf "==> $(gettext "ERROR:") ${mesg}\n" "$@" >&2 -} - -# print usage instructions -usage() { - printf "repo-add, repo-remove (pacman) %s\n\n" "$myver" - printf "$(gettext "Usage: repo-add [-d] [-f] [-q] ...\n")" - printf "$(gettext "Usage: repo-remove [-q] ...\n\n")" - printf "$(gettext "\ -repo-add will update a package database by reading a package file.\n\ -Multiple packages to add can be specified on the command line.\n\n")" - printf "$(gettext "\ -repo-remove will update a package database by removing the package name\n\ -specified on the command line from the given repo database. Multiple\n\ -packages to remove can be specified on the command line.\n\n")" - printf "$(gettext "\ -Use the -q/--quiet flag to minimize output to basic messages, warnings,\n\ -and errors.\n\n")" - printf "$(gettext "\ -Use the -d/--delta flag to automatically generate and add a delta file\n\ -between the old entry and the new one, if the old package file is found\n\ -next to the new one.\n\n")" - printf "$(gettext "\ -Use the -f/--files flag to update a database including file entries.\n\n")" - echo "$(gettext "Example: repo-add /path/to/repo.db.tar.gz pacman-3.0.0.pkg.tar.gz")" - echo "$(gettext "Example: repo-remove /path/to/repo.db.tar.gz kernel26")" -} - -version() { - printf "repo-add, repo-remove (pacman) %s\n\n" "$myver" - printf "$(gettext "\ -Copyright (C) 2006-2008 Aaron Griffin .\n\ -Copyright (c) 2007-2008 Dan McGee .\n\n\ -This is free software; see the source for copying conditions.\n\ -There is NO WARRANTY, to the extent permitted by law.\n")" -} - -# write a list entry -# arg1 - Entry name -# arg2 - List -# arg3 - File to write to -write_list_entry() { - if [[ -n $2 ]]; then - echo "%$1%" >>$3 - echo -e $2 >>$3 - fi -} - -find_pkgentry() -{ - local pkgname=$1 - local pkgentry - for pkgentry in $tmpdir/$pkgname*; do - name=${pkgentry##*/} - if [[ ${name%-*-*} = $pkgname ]]; then - echo $pkgentry - return 0 - fi - done - return 1 -} - -# Get the package name from the delta filename -get_delta_pkgname() { - local tmp - - tmp=${1##*/} - echo ${tmp%-*-*_to*} -} - -# write a delta entry -# arg1 - path to delta file -db_write_delta() -{ - deltafile="$1" - pkgname="$(get_delta_pkgname $deltafile)" - - pkgentry=$(find_pkgentry $pkgname) - if [[ -z $pkgentry ]]; then - error "$(gettext "No database entry for package '%s'.")" "$pkgname" - return 1 - fi - deltas="$pkgentry/deltas" - if [[ ! -f $deltas ]]; then - echo -e "%DELTAS%" >$deltas - fi - # get md5sum and compressed size of package - md5sum="$(openssl dgst -md5 "$deltafile")" - md5sum="${md5sum##* }" - csize=$(stat -L -c %s "$deltafile") - - oldfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (source)" | sed 's/.*: *//') - newfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (output)" | sed 's/.*: *//') - - if grep -q "$oldfile.*$newfile" $deltas; then - sed -i.backup "/$oldfile.*$newfile/d" $deltas && rm -f $deltas.backup - fi - msg2 "$(gettext "Adding 'deltas' entry : %s -> %s")" "$oldfile" "$newfile" - echo ${deltafile##*/} $md5sum $csize $oldfile $newfile >> $deltas - - return 0 -} # end db_write_delta - -# remove a delta entry -# arg1 - path to delta file -db_remove_delta() -{ - deltafile="$1" - filename=${deltafile##*/} - pkgname="$(get_delta_pkgname $deltafile)" - - pkgentry=$(find_pkgentry $pkgname) - if [[ -z $pkgentry ]]; then - return 1 - fi - deltas="$pkgentry/deltas" - if [[ ! -f $deltas ]]; then - return 1 - fi - if grep -q "$filename" $deltas; then - sed -i.backup "/$filename/d" $deltas && rm -f $deltas.backup - msg2 "$(gettext "Removing existing entry '%s'...")" "$filename" - return 0 - fi - - return 1 -} # end db_remove_delta - -# write an entry to the pacman database -# arg1 - path to package -db_write_entry() -{ - # blank out all variables - local pkgfile="$1" - local pkgname pkgver pkgdesc csize size md5sum url arch builddate packager \ - _groups _licenses _replaces _depends _conflicts _provides _optdepends - - local OLDIFS="$IFS" - # IFS (field separator) is only the newline character - IFS=" -" - - # read info from the zipped package - local line var val - for line in $(bsdtar -xOqf "$pkgfile" .PKGINFO | - grep -v '^#' | sed 's|\(\w*\)\s*=\s*\(.*\)|\1 \2|'); do - # bash awesomeness here- var is always one word, val is everything else - var=${line%% *} - val=${line#* } - declare $var="$val" - case "$var" in - group) _groups="$_groups$group\n" ;; - license) _licenses="$_licenses$license\n" ;; - replaces) _replaces="$_replaces$replaces\n" ;; - depend) _depends="$_depends$depend\n" ;; - conflict) _conflicts="$_conflicts$conflict\n" ;; - provides) _provides="$_provides$provides\n" ;; - optdepend) _optdepends="$_optdepends$optdepend\n" ;; - esac - done - - IFS=$OLDIFS - - # get md5sum and compressed size of package - md5sum="$(openssl dgst -md5 "$pkgfile")" - md5sum="${md5sum##* }" - csize=$(stat -L -c %s "$pkgfile") - - # ensure $pkgname and $pkgver variables were found - if [[ -z $pkgname || -z $pkgver ]]; then - error "$(gettext "Invalid package file '%s'.")" "$pkgfile" - return 1 - fi - - pushd "$tmpdir" >/dev/null - if [[ -d $pkgname-$pkgver ]]; then - warning "$(gettext "An entry for '%s' already existed")" "$pkgname-$pkgver" - else - if (( DELTA )); then - pkgentry=$(find_pkgentry $pkgname) - if [[ -n $pkgentry ]]; then - local oldfilename=$(grep -A1 FILENAME $pkgentry/desc | tail -n1) - local oldfile="$(dirname $1)/$oldfilename" - fi - fi - fi - - # remove an existing entry if it exists, ignore failures - db_remove_entry "$pkgname" - - # create package directory - mkdir "$pkgname-$pkgver" - pushd "$pkgname-$pkgver" >/dev/null - - # restore an eventual deltas file - [[ -f ../$pkgname.deltas ]] && mv "../$pkgname.deltas" deltas - - # create desc entry - msg2 "$(gettext "Creating '%s' db entry...")" 'desc' - echo -e "%FILENAME%\n$(basename "$1")\n" >>desc - echo -e "%NAME%\n$pkgname\n" >>desc - [[ -n $pkgbase ]] && echo -e "%BASE%\n$pkgbase\n" >>desc - echo -e "%VERSION%\n$pkgver\n" >>desc - [[ -n $pkgdesc ]] && echo -e "%DESC%\n$pkgdesc\n" >>desc - write_list_entry "GROUPS" "$_groups" "desc" - [[ -n $csize ]] && echo -e "%CSIZE%\n$csize\n" >>desc - [[ -n $size ]] && echo -e "%ISIZE%\n$size\n" >>desc - - # compute checksums - msg2 "$(gettext "Computing md5 checksums...")" - echo -e "%MD5SUM%\n$md5sum\n" >>desc - - [[ -n $url ]] && echo -e "%URL%\n$url\n" >>desc - write_list_entry "LICENSE" "$_licenses" "desc" - [[ -n $arch ]] && echo -e "%ARCH%\n$arch\n" >>desc - [[ -n $builddate ]] && echo -e "%BUILDDATE%\n$builddate\n" >>desc - [[ -n $packager ]] && echo -e "%PACKAGER%\n$packager\n" >>desc - write_list_entry "REPLACES" "$_replaces" "desc" - - # create depends entry - msg2 "$(gettext "Creating '%s' db entry...")" 'depends' - # create the file even if it will remain empty - touch "depends" - write_list_entry "DEPENDS" "$_depends" "depends" - write_list_entry "CONFLICTS" "$_conflicts" "depends" - write_list_entry "PROVIDES" "$_provides" "depends" - write_list_entry "OPTDEPENDS" "$_optdepends" "depends" - - popd >/dev/null - popd >/dev/null - - # create files file if wanted - if (( WITHFILES )); then - msg2 "$(gettext "Creating '%s' db entry...")" 'files' - local files_path="$tmpdir/$pkgname-$pkgver/files" - echo "%FILES%" >$files_path - bsdtar --exclude='.*' -tf "$pkgfile" >>$files_path - fi - - # create a delta file - if (( DELTA )); then - if [[ -n $oldfilename ]]; then - if [[ -f $oldfile ]]; then - delta=$(pkgdelta -q $oldfile $1) - if [[ -f $delta ]]; then - db_write_delta $delta - fi - else - warning "$(gettext "Old package file not found: %s")" "$oldfilename" - fi - fi - fi - - # Extracts licenses to a common license dir - msg "Extracting license" - if bsdtar -xOf ${pkgfile} .PKGINFO | grep "license" | grep "custom" ; then - if [ -d ${LICENSESDIR}/${pkgname} ]; then - rm -r ${LICENSESDIR}/${pkgname} - fi - - # Change dir to licenses, and extract them stripping the first part of the path - bsdtar -C ${LICENSESDIR}/ --include="usr/share/licenses/" \ - --strip-components 3 -xf ${pkgfile} >/dev/null 2>&1 - - if [ $? -ne 0 ]; then - warning "This package doesn't contain a license dir" - fi - fi - - return 0 -} # end db_write_entry - -# remove existing entries from the DB -# arg1 - package name -db_remove_entry() { - local pkgname=$1 - local notfound=1 - local pkgentry=$(find_pkgentry $pkgname) - while [[ -n $pkgentry ]]; do - notfound=0 - if [[ -f $pkgentry/deltas ]]; then - mv "$pkgentry/deltas" "$tmpdir/$pkgname.deltas" - fi - msg2 "$(gettext "Removing existing entry '%s'...")" \ - "$(basename $pkgentry)" - rm -rf $pkgentry - pkgentry=$(find_pkgentry $pkgname) - done - - msg "Removing license" - if [ -d ${LICENSESDIR}/${pkgname} ]; then - rm -r ${LICENSESDIR}/${pkgname} - fi - - return $notfound -} # end db_remove_entry - -check_repo_db() -{ - # check lock file - if ( set -o noclobber; echo "$$" > "$LOCKFILE") 2> /dev/null; then - CLEAN_LOCK=1 - else - error "$(gettext "Failed to acquire lockfile: %s.")" "$LOCKFILE" - [[ -f $LOCKFILE ]] && error "$(gettext "Held by process %s")" "$(cat $LOCKFILE)" - exit 1 - fi - - if [[ -f $REPO_DB_FILE ]]; then - # there are two situations we can have here- a DB with some entries, - # or a DB with no contents at all. - if ! bsdtar -tqf "$REPO_DB_FILE" '*/desc' >/dev/null 2>&1; then - # check empty case - if [[ -n $(bsdtar -tqf "$REPO_DB_FILE" '*' 2>/dev/null) ]]; then - error "$(gettext "Repository file '%s' is not a proper pacman database.")" "$REPO_DB_FILE" - exit 1 - fi - fi - msg "$(gettext "Extracting database to a temporary location...")" - bsdtar -xf "$REPO_DB_FILE" -C "$tmpdir" - else - case "$cmd" in - repo-remove) - error "$(gettext "Repository file '%s' was not found.")" "$REPO_DB_FILE" - exit 1 - ;; - repo-add) - # check if the file can be created (write permission, directory existence, etc) - if ! touch "$REPO_DB_FILE"; then - error "$(gettext "Repository file '%s' could not be created.")" "$REPO_DB_FILE" - exit 1 - fi - rm -f "$REPO_DB_FILE" - ;; - esac - fi -} - -add() -{ - if [[ ! -f $1 ]]; then - error "$(gettext "File '%s' not found.")" "$1" - return 1 - fi - - if [[ ${1##*.} == "delta" ]]; then - deltafile=$1 - msg "$(gettext "Adding delta '%s'")" "$deltafile" - if ! type xdelta3 &>/dev/null; then - error "$(gettext "Cannot find the xdelta3 binary! Is xdelta3 installed?")" - exit 1 - fi - if db_write_delta "$deltafile"; then - return 0 - else - return 1 - fi - fi - - pkgfile=$1 - if ! bsdtar -tqf "$pkgfile" .PKGINFO >/dev/null 2>&1; then - error "$(gettext "'%s' is not a package file, skipping")" "$pkgfile" - return 1 - fi - - msg "$(gettext "Adding package '%s'")" "$pkgfile" - - db_write_entry "$pkgfile" -} - -remove() -{ - if [[ ${1##*.} == "delta" ]]; then - deltafile=$1 - msg "$(gettext "Searching for delta '%s'...")" "$deltafile" - if db_remove_delta "$deltafile"; then - return 0 - else - error "$(gettext "Delta matching '%s' not found.")" "$deltafile" - return 1 - fi - fi - - pkgname=$1 - msg "$(gettext "Searching for package '%s'...")" "$pkgname" - - if db_remove_entry "$pkgname"; then - rm -f "$tmpdir/$pkgname.deltas" - return 0 - else - error "$(gettext "Package matching '%s' not found.")" "$pkgname" - return 1 - fi -} - -trap_exit() -{ - echo - error "$@" - exit 1 -} - -clean_up() { - local exit_code=$? - - [[ -d $tmpdir ]] && rm -rf "$tmpdir" - (( CLEAN_LOCK )) && [[ -f $LOCKFILE ]] && rm -f "$LOCKFILE" - - exit $exit_code -} - -# PROGRAM START - -# determine whether we have gettext; make it a no-op if we do not -if ! type gettext &>/dev/null; then - gettext() { - echo "$@" - } -fi - -case "$1" in - -h|--help) usage; exit 0;; - -V|--version) version; exit 0;; -esac - -# figure out what program we are -cmd="$(basename $0)" -if [[ $cmd != "repo-add" && $cmd != "repo-remove" ]]; then - error "$(gettext "Invalid command name '%s' specified.")" "$cmd" - exit 1 -fi - -tmpdir=$(mktemp -d /tmp/repo-tools.XXXXXXXXXX) || (\ - error "$(gettext "Cannot create temp directory for database building.")"; \ - exit 1) - -trap 'clean_up' EXIT -trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT -trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT -trap 'trap_exit "$(gettext "An unknown error has occured. Exiting...")"' ERR - -success=0 -# parse arguments -for arg in "$@"; do - case "$arg" in - -q|--quiet) QUIET=1;; - -d|--delta) DELTA=1;; - -f|--files) WITHFILES=1;; - *) - if [[ -z $REPO_DB_FILE ]]; then - REPO_DB_FILE="$arg" - LOCKFILE="$REPO_DB_FILE.lck" - check_repo_db - else - case "$cmd" in - repo-add) add $arg && success=1 ;; - repo-remove) remove $arg && success=1 ;; - esac - fi - ;; - esac -done - -# if at least one operation was a success, re-zip database -if (( success )); then - msg "$(gettext "Creating updated database file '%s'")" "$REPO_DB_FILE" - - case "$REPO_DB_FILE" in - *tar.gz) TAR_OPT="z" ;; - *tar.bz2) TAR_OPT="j" ;; - *tar.xz) TAR_OPT="J" ;; - *) warning "$(gettext "'%s' does not have a valid archive extension.")" \ - "$REPO_DB_FILE" ;; - esac - - filename=$(basename "$REPO_DB_FILE") - - pushd "$tmpdir" >/dev/null - if [[ -n $(ls) ]]; then - bsdtar -c${TAR_OPT}f "$filename" * - else - # we have no packages remaining? zip up some emptyness - warning "$(gettext "No packages remain, creating empty database.")" - bsdtar -c${TAR_OPT}f "$filename" -T /dev/null - fi - popd >/dev/null - - [[ -f $REPO_DB_FILE ]] && mv -f "$REPO_DB_FILE" "${REPO_DB_FILE}.old" - [[ -f $tmpdir/$filename ]] && mv "$tmpdir/$filename" "$REPO_DB_FILE" - dblink="${REPO_DB_FILE%.tar.*}" - target=${REPO_DB_FILE##*/} - ln -sf "$target" "$dblink" 2>/dev/null || \ - ln -f "$target" "$dblink" 2>/dev/null || \ - cp "$REPO_DB_FILE" "$dblink" -else - msg "$(gettext "No packages modified, nothing to do.")" - exit 1 -fi - -exit 0 -# vim: set ts=2 sw=2 noet: -- cgit v1.2.3-2-g168b From 00b1be5472acddf258f948719714ec1c6f3cdab1 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 1 Jan 2014 22:07:36 -0500 Subject: local_config.example: update to reflect what is on repo --- local_config.example | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/local_config.example b/local_config.example index daa6874..ec22d84 100644 --- a/local_config.example +++ b/local_config.example @@ -1,8 +1,13 @@ #/bin/bash # as a hint to text editors -_paraboladir=~/parabolagnulinux.org +_paraboladir=/srv/http/repo/public # db-sync -mirror="mirrors.eu.kernel.org" +#mirror="mirrors.uk2.net" +mirror="mirrors.kernel.org" +#mirror="mirror.umd.edu" +#mirror="archlinux.c3sl.ufpr.br" +#mirror="mirror.us.leaseweb.net" +#mirror="mirror.de.leaseweb.net" mirrorpath="mirrors/archlinux" # mkrepo -- cgit v1.2.3-2-g168b From b57c78930fec71dc6c5ef8eaa5148a22f75a1fff Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 1 Jan 2014 22:08:39 -0500 Subject: go ahead and track local_config --- .gitignore | 3 +-- local_config | 20 ++++++++++++++++++++ local_config.example | 20 -------------------- 3 files changed, 21 insertions(+), 22 deletions(-) create mode 100644 local_config delete mode 100644 local_config.example diff --git a/.gitignore b/.gitignore index dd17455..98a5228 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,9 @@ *~ *.pyc -local_config /config.local test/packages/*/*.pkg.tar.?z \#*# .#* yftime src* -pkg* \ No newline at end of file +pkg* diff --git a/local_config b/local_config new file mode 100644 index 0000000..ec22d84 --- /dev/null +++ b/local_config @@ -0,0 +1,20 @@ +#/bin/bash # as a hint to text editors +_paraboladir=/srv/http/repo/public + +# db-sync +#mirror="mirrors.uk2.net" +mirror="mirrors.kernel.org" +#mirror="mirror.umd.edu" +#mirror="archlinux.c3sl.ufpr.br" +#mirror="mirror.us.leaseweb.net" +#mirror="mirror.de.leaseweb.net" +mirrorpath="mirrors/archlinux" + +# mkrepo +repodir=${_paraboladir}/repo + +# yf-update +blacklist=${_paraboladir}/docs/blacklist.txt +whitelist=${_paraboladir}/docs/whitelist.txt + +unset _paraboladir diff --git a/local_config.example b/local_config.example deleted file mode 100644 index ec22d84..0000000 --- a/local_config.example +++ /dev/null @@ -1,20 +0,0 @@ -#/bin/bash # as a hint to text editors -_paraboladir=/srv/http/repo/public - -# db-sync -#mirror="mirrors.uk2.net" -mirror="mirrors.kernel.org" -#mirror="mirror.umd.edu" -#mirror="archlinux.c3sl.ufpr.br" -#mirror="mirror.us.leaseweb.net" -#mirror="mirror.de.leaseweb.net" -mirrorpath="mirrors/archlinux" - -# mkrepo -repodir=${_paraboladir}/repo - -# yf-update -blacklist=${_paraboladir}/docs/blacklist.txt -whitelist=${_paraboladir}/docs/whitelist.txt - -unset _paraboladir -- cgit v1.2.3-2-g168b From 7e93af9ef087cd048dd42cef71328ec7f75e885c Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 5 Jan 2014 04:21:24 +0000 Subject: fix use of $0 in db-cleanup --- db-cleanup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db-cleanup b/db-cleanup index 904c06e..841800b 100755 --- a/db-cleanup +++ b/db-cleanup @@ -43,7 +43,7 @@ for _repo in ${PKGREPOS[@]}; do bsdtar tf "${dbfile}" | \ cut -d'/' -f1 | \ sort -u | \ - sed "s|$|*|" >> /tmp/$0.$$.filter + sed "s|$|*|" >> /tmp/${0##*/}.$$.filter done done @@ -54,7 +54,7 @@ for POOL in ${PKGPOOLS[@]} ${SRCPOOLS[@]}; do msg2 "${POOL}" rsync ${EXTRAFLAGS} -va --delete-excluded \ - --include-from="/tmp/$0.$$.filter" \ + --include-from="/tmp/${0##*/}.$$.filter" \ --exclude="*" \ ${FTP_BASE}/${POOL}/ \ ${FTP_BASE}/${POOL}/ -- cgit v1.2.3-2-g168b From ba21dd9e342bd640b1bbaa82ebf015932390b18e Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 5 Jan 2014 04:21:52 +0000 Subject: db-update: add hook for publishing generated sources --- db-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-update b/db-update index 0359697..2fa23af 100755 --- a/db-update +++ b/db-update @@ -83,4 +83,4 @@ for repo in ${repos[@]}; do done done - +cp -rviT "${STAGING}/other/" "${FTP_BASE}/other/" -- cgit v1.2.3-2-g168b From d4f3035779dae2efc5c5baa527ebd82404207290 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 5 Jan 2014 04:22:51 +0000 Subject: go ahead and track local_config --- .gitignore | 3 +-- local_config | 31 +++++++++++++++++++++++++++++++ local_config.example | 26 -------------------------- 3 files changed, 32 insertions(+), 28 deletions(-) create mode 100644 local_config delete mode 100644 local_config.example diff --git a/.gitignore b/.gitignore index dd17455..98a5228 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,9 @@ *~ *.pyc -local_config /config.local test/packages/*/*.pkg.tar.?z \#*# .#* yftime src* -pkg* \ No newline at end of file +pkg* diff --git a/local_config b/local_config new file mode 100644 index 0000000..8f3946a --- /dev/null +++ b/local_config @@ -0,0 +1,31 @@ +# Mirror options +#mirror="mirrors.uk2.net" +mirror="mirrors.kernel.org" +#mirror="mirror.umd.edu" +#mirror="archlinux.c3sl.ufpr.br" +#mirror="mirror.us.leaseweb.net" +#mirror="mirror.de.leaseweb.net" +mirrorpath="archlinux" + +# Directories: they should end without / +paraboladir=/srv/http/repo/public +tempdir=/tmp +archdb=${tempdir}/db +docs_dir=${paraboladir}/docs +repodir=${paraboladir} +licenses_dir=${docs_dir}/pending_licenses +# End Directories + +# Files +logname=${paraboladir}/log/$(date -u +%Y%m%d-%H:%M)-repo-maintainer.log +rsout_file=${tempdir}/rsout +rsync_not_needed=${tempdir}/rsync_not_needed + +rsync_blacklist=${docs_dir}/rsyncBlacklist + +blacklist=${docs_dir}/blacklist.txt +whitelist=${docs_dir}/whitelist.txt + +# Rsync commands +rsync_list_command="rsync -rptgoL --exclude='*.abs.tar.*' --list-only --no-motd " +rsync_update_command="rsync -vrptgoL --exclude='*.abs.tar.*' --no-motd " diff --git a/local_config.example b/local_config.example deleted file mode 100644 index 2280cc2..0000000 --- a/local_config.example +++ /dev/null @@ -1,26 +0,0 @@ -# Mirror options -mirror="mirrors.eu.kernel.org" -mirrorpath="mirrors/archlinux" - -# Directories: they should end without / -paraboladir=~/parabolagnulinux.org -tempdir=~/tmp -archdb=${tempdir}/db -docs_dir=${paraboladir}/docs -repodir=${paraboladir}/repo -licenses_dir=${docs_dir}/pending_licenses -# End Directories - -# Files -logname=${paraboladir}/log/$(date -u +%Y%m%d-%H:%M)-repo-maintainer.log -rsout_file=${tempdir}/rsout -rsync_not_needed=${tempdir}/rsync_not_needed - -rsync_blacklist=${docs_dir}/rsyncBlacklist - -blacklist=${docs_dir}/blacklist.txt -whitelist=${docs_dir}/whitelist.txt - -# Rsync commands -rsync_list_command="rsync -rptgoL --exclude='*.abs.tar.*' --list-only --no-motd " -rsync_update_command="rsync -rptgoL --exclude='*.abs.tar.*' --no-motd " -- cgit v1.2.3-2-g168b From 91f6039acb4ba915a3ca3fc366a7159656ec0dc1 Mon Sep 17 00:00:00 2001 From: Parabola Date: Mon, 6 Jan 2014 04:30:22 +0000 Subject: fix things --- db-remove | 2 +- db-update | 21 +++++++++++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/db-remove b/db-remove index a71bbab..f0785e5 100755 --- a/db-remove +++ b/db-remove @@ -33,7 +33,7 @@ for pkgbase in ${pkgbases[@]}; do if [ -d "${SVNREPO}/$repo/$pkgbase" ]; then remove_pkgs=($(. "${SVNREPO}/$repo/$pkgbase/PKGBUILD"; echo ${pkgname[@]})) else - warning "$pkgbase not found in $svnrepo" + warning "$pkgbase not found in ABS(libre)" warning "Removing only $pkgbase from the repo" warning "If it was a split package you have to remove the others yourself!" remove_pkgs[${#remove_pkgs[*]}]=$pkgbase diff --git a/db-update b/db-update index 2fa23af..2d4f28a 100755 --- a/db-update +++ b/db-update @@ -9,7 +9,7 @@ if [ $# -ge 1 ]; then fi # Find repos with packages to release -repos=($(find "${STAGING}" -mindepth 1 -maxdepth 1 -type d ! -empty -printf '%f ' 2>/dev/null)) +repos=($(find "${STAGING}" -mindepth 1 -maxdepth 1 -type d ! -empty ! -name other -printf '%f ' 2>/dev/null)) if [ $? -ge 1 ]; then die "Could not read ${STAGING}" fi @@ -39,10 +39,10 @@ for repo in ${repos[@]}; do die "Package ${repo}/$(basename ${pkg}) already exists in another repository" fi done - # This is fucking obnoxious -# if ! check_splitpkgs ${repo} ${pkgs[@]}; then -# die "Missing split packages for ${repo}" -# fi + # This is fucking obnoxious + #if ! check_splitpkgs ${repo} ${pkgs[@]}; then + # die "Missing split packages for ${repo}" + #fi else die "Could not read ${STAGING}" fi @@ -83,4 +83,13 @@ for repo in ${repos[@]}; do done done -cp -rviT "${STAGING}/other/" "${FTP_BASE}/other/" +cd "${STAGING}" +while read -r file; do + pub="${FTP_BASE}/${file}" + if [[ -f $pub ]]; then + warning "file already exists: %s" "${file}" + else + mkdir -p -- "${pub%/*}" + mv -vn "$file" "$pub" + fi +done < <(find other -type f) -- cgit v1.2.3-2-g168b From 6682aefeafb8d30899e464122864f56318ad7640 Mon Sep 17 00:00:00 2001 From: Parabola Date: Wed, 8 Jan 2014 05:20:12 +0000 Subject: Don't error about permissions on empty staging repos (feature #460) --- db-update | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db-update b/db-update index 2d4f28a..b9b8015 100755 --- a/db-update +++ b/db-update @@ -23,11 +23,11 @@ done # check if packages are valid for repo in ${repos[@]}; do - if ! check_repo_permission "${repo}"; then - die "You don't have permission to update packages in ${repo}" - fi pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXT})) if [ $? -eq 0 ]; then + if [ ${#pkgs[@]} -gt 0 ] && ! check_repo_permission "${repo}"; then + die "You don't have permission to update packages in ${repo}" + fi for pkg in ${pkgs[@]}; do if [ -h "${pkg}" ]; then die "Package ${repo}/$(basename ${pkg}) is a symbolic link" @@ -92,4 +92,4 @@ while read -r file; do mkdir -p -- "${pub%/*}" mv -vn "$file" "$pub" fi -done < <(find other -type f) +done < <(find other sources -type f) -- cgit v1.2.3-2-g168b From 725ab5d12375dd593c375b1a494021bb96135d4f Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 00:30:36 -0500 Subject: db-update: remove 'other' from repo blacklist now that issue #460 is added --- db-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-update b/db-update index b9b8015..1fddb8a 100755 --- a/db-update +++ b/db-update @@ -9,7 +9,7 @@ if [ $# -ge 1 ]; then fi # Find repos with packages to release -repos=($(find "${STAGING}" -mindepth 1 -maxdepth 1 -type d ! -empty ! -name other -printf '%f ' 2>/dev/null)) +repos=($(find "${STAGING}" -mindepth 1 -maxdepth 1 -type d ! -empty -printf '%f ' 2>/dev/null)) if [ $? -ge 1 ]; then die "Could not read ${STAGING}" fi -- cgit v1.2.3-2-g168b From 08496a8bd13ffc7228feac53accb93b063c4b358 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 14:55:11 -0500 Subject: config: fix typo --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 3b7c13a..c6fca1b 100644 --- a/config +++ b/config @@ -1,4 +1,4 @@ -#/bin/bash # as a hint to text editors +#!/bin/bash # as a hint to text editors FTP_BASE="/srv/http/repo/public" SVNREPO="/var/abs" -- cgit v1.2.3-2-g168b From fedfd6f27bff4fddbaa86fb3881f591746a47fd2 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 15:15:16 -0500 Subject: createrepos: fix quoting --- createrepos | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/createrepos b/createrepos index cd17e4e..8da2455 100755 --- a/createrepos +++ b/createrepos @@ -3,6 +3,6 @@ source "$(dirname "$(readlink -e "$0")")/config" -mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR} ${STAGING}" +mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" "$(dirname "$(readlink -e "$0")")/create-repo" "${PKGREPOS[@]}" -- cgit v1.2.3-2-g168b From 4a0e623b0c8162b466695be7f5091c76b8c0b63b Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 16:14:55 -0500 Subject: db-functions: use ${array[*]} when appropriate --- db-functions | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db-functions b/db-functions index 0b59a53..efb6172 100644 --- a/db-functions +++ b/db-functions @@ -424,9 +424,9 @@ arch_repo_add() { # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ - || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" + || error "repo-add ${repo}${DBEXT} ${pkgs[*]}" /usr/bin/repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ - || error "repo-add -f ${repo}${FILESEXT} ${pkgs[@]}" + || error "repo-add -f ${repo}${FILESEXT} ${pkgs[*]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" @@ -445,9 +445,9 @@ arch_repo_remove() { return 1 fi /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ - || error "repo-remove ${dbfile} ${pkgs[@]}" + || error "repo-remove ${dbfile} ${pkgs[*]}" /usr/bin/repo-remove -q "${filesfile}" ${pkgs[@]} \ - || error "repo-remove ${filesfile} ${pkgs[@]}" + || error "repo-remove ${filesfile} ${pkgs[*]}" set_repo_permission "${repo}" "${arch}" REPO_MODIFIED=1 -- cgit v1.2.3-2-g168b From e7d2dcac7cf857fdccd82bec2bfc2a7d8e6b85c6 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 16:53:56 -0500 Subject: Normalize to load ./config before loading ./db-functions --- create-repo | 2 +- cron-jobs/repo-sanity-check | 2 +- cron-jobs/sourceballs2 | 2 +- db-check-nonfree | 2 +- db-list-unsigned-packages | 2 +- repo-restore-to-normal | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/create-repo b/create-repo index 24b890d..21a2a9c 100755 --- a/create-repo +++ b/create-repo @@ -1,8 +1,8 @@ #!/bin/bash # Creates repository structure -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -eq 0 ]; then msg "Usage: ${0##*/} repo1 [repo2 ... repoX]" diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index 2aa7892..ee4c061 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -1,8 +1,8 @@ #!/bin/bash # Solves issue165 -. "$(dirname "$(readlink -e "$0")")/../db-functions" . "$(dirname "$(readlink -e "$0")")/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" # Traverse all repos for _repo in ${PKGREPOS[@]}; do diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index bbe227d..1432bdf 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -5,8 +5,8 @@ # Remove the old sourceballs dirname="$(dirname "$(readlink -e "$0")")" -. "${dirname}/../db-functions" . "${dirname}/../config" +. "${dirname}/../db-functions" . "${MAKEPKGCONF}" pushd "${WORKDIR}" >/dev/null diff --git a/db-check-nonfree b/db-check-nonfree index 661daa6..5cb7f6f 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -ge 1 ]; then warning "Calling ${0##*/} with a specific repository is not supported" diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages index 985d1c0..5105096 100755 --- a/db-list-unsigned-packages +++ b/db-list-unsigned-packages @@ -20,8 +20,8 @@ set -e # unsigned packages available for architecture $1 and specified for # architecture $2 (usually $1 or any, default is to list all). -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 1 ]; then msg "usage: ${0##*/} " diff --git a/repo-restore-to-normal b/repo-restore-to-normal index 3636920..3fe4816 100755 --- a/repo-restore-to-normal +++ b/repo-restore-to-normal @@ -1,8 +1,8 @@ #!/bin/bash # Solves issue165 -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" CLEANUP_DESTDIR=/home/parabolavnx/repo/pool/restore PKGREPOS=(community) -- cgit v1.2.3-2-g168b From 39fbf0d8d3cdc666912c597d41d5b6a70fc0c725 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 20:53:38 -0500 Subject: Fix some array quoting. --- create-repo | 4 ++-- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 22 ++++++++++---------- cron-jobs/ftpdir-cleanup | 12 +++++------ cron-jobs/repo-sanity-check | 4 ++-- cron-jobs/sourceballs | 26 ++++++++++++------------ cron-jobs/sourceballs2 | 4 ++-- db-check-nonfree | 22 ++++++++++---------- db-functions | 30 ++++++++++++++-------------- db-list-unsigned-packages | 2 +- db-move | 27 ++++++++++++------------- db-remove | 6 +++--- db-sync | 5 ++--- db-update | 26 ++++++++++++------------ get-repos | 6 +++--- repo-restore-to-normal | 6 +++--- 15 files changed, 100 insertions(+), 102 deletions(-) diff --git a/create-repo b/create-repo index 21a2a9c..1ec9798 100755 --- a/create-repo +++ b/create-repo @@ -10,12 +10,12 @@ if [ $# -eq 0 ]; then fi msg "Creating repos..." -for _repo in $@; do +for _repo in "$@"; do msg2 "Creating [${_repo}]" mkdir -p "${FTP_BASE}/staging/${_repo}" || \ error "Failed creating staging dir" - for _arch in ${ARCHES[@]}; do + for _arch in "${ARCHES[@]}"; do mkdir -p "${FTP_BASE}/${_repo}/os/${_arch}" || \ error "Failed creating ${_arch} dir" done diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index 3f92169..c8d8618 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -6,18 +6,18 @@ exit() { return; } splitpkg_overrides=('depends' 'optdepends' 'provides' 'conflicts') -variables=('pkgname' 'pkgbase' 'epoch' 'pkgver' 'pkgrel' 'makedepends' 'arch' ${splitpkg_overrides[@]}) +variables=('pkgname' 'pkgbase' 'epoch' 'pkgver' 'pkgrel' 'makedepends' 'arch' "${splitpkg_overrides[@]}") readonly -a variables splitpkg_overrides backup_package_variables() { - for var in ${splitpkg_overrides[@]}; do + for var in "${splitpkg_overrides[@]}"; do indirect="${var}_backup" eval "${indirect}=(\${$var[@]})" done } restore_package_variables() { - for var in ${splitpkg_overrides[@]}; do + for var in "${splitpkg_overrides[@]}"; do indirect="${var}_backup" if [ -n "${!indirect}" ]; then eval "${var}=(\${$indirect[@]})" @@ -42,31 +42,31 @@ print_info() { if [ -n "$arch" ]; then echo "%ARCH%" - for i in ${arch[@]}; do echo $i; done + for i in "${arch[@]}"; do echo $i; done echo "" fi if [ -n "$depends" ]; then echo "%DEPENDS%" - for i in ${depends[@]}; do + for i in "${depends[@]}"; do echo $i done echo "" fi if [ -n "$makedepends" ]; then echo "%MAKEDEPENDS%" - for i in ${makedepends[@]}; do + for i in "${makedepends[@]}"; do echo $i done echo "" fi if [ -n "$conflicts" ]; then echo "%CONFLICTS%" - for i in ${conflicts[@]}; do echo $i; done + for i in "${conflicts[@]}"; do echo $i; done echo "" fi if [ -n "$provides" ]; then echo "%PROVIDES%" - for i in ${provides[@]}; do echo $i; done + for i in "${provides[@]}"; do echo $i; done echo "" fi } @@ -75,7 +75,7 @@ source_pkgbuild() { ret=0 dir=$1 pkgbuild=$dir/PKGBUILD - for var in ${variables[@]}; do + for var in "${variables[@]}"; do unset ${var} done source $pkgbuild &>/dev/null || ret=$? @@ -88,7 +88,7 @@ source_pkgbuild() { if [ "${#pkgname[@]}" -gt "1" ]; then pkgbase=${pkgbase:-${pkgname[0]}} - for pkg in ${pkgname[@]}; do + for pkg in "${pkgname[@]}"; do if [ "$(type -t package_${pkg})" != "function" ]; then echo -e "%INVALID%\n$pkgbuild\n" return 1 @@ -98,7 +98,7 @@ source_pkgbuild() { while IFS= read -r line; do var=${line%%=*} var="${var#"${var%%[![:space:]]*}"}" # remove leading whitespace characters - for realvar in ${variables[@]}; do + for realvar in "${variables[@]}"; do if [ "$var" == "$realvar" ]; then eval $line break diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index e42a1a8..ad2e7f9 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -24,16 +24,16 @@ clean_pkg() { script_lock -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do repo_lock ${repo} ${arch} || exit 1 done done ${CLEANUP_DRYRUN} && warning 'dry run mode is active' -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue fi @@ -87,8 +87,8 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then done fi -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do repo_unlock ${repo} ${arch} done done diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index ee4c061..9d351df 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -5,7 +5,7 @@ . "$(dirname "$(readlink -e "$0")")/../db-functions" # Traverse all repos -for _repo in ${PKGREPOS[@]}; do +for _repo in "${PKGREPOS[@]}"; do msg "Cleaning up [${_repo}]" # Find all pkgnames on this repo's abs @@ -19,7 +19,7 @@ for _repo in ${PKGREPOS[@]}; do >/dev/null 2>&1 # also cleanup package functions - for _pkg in ${pkgname[@]}; do + for _pkg in "${pkgname[@]}"; do unset package_${pkg} >/dev/null 2>&1 done diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 1542499..329e135 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -7,8 +7,8 @@ pushd "${WORKDIR}" >/dev/null script_lock -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do repo_lock ${repo} ${arch} || exit 1 done done @@ -18,8 +18,8 @@ renice +10 -p $$ > /dev/null # Create a readable file for each repo with the following format # - [ ] -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do # Repo does not exist; skip it if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue @@ -39,8 +39,8 @@ for repo in ${PKGREPOS[@]}; do done | sort -u > "${WORKDIR}/db-${repo}" done -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do repo_unlock ${repo} ${arch} done done @@ -49,15 +49,15 @@ done find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" # Check for all packages if we need to build a source package -for repo in ${PKGREPOS[@]}; do +for repo in "${PKGREPOS[@]}"; do newpkgs=() failedpkgs=() while read line; do - pkginfo=(${line}) + pkginfo=("${line}") pkgbase=${pkginfo[0]} pkgver=${pkginfo[1]} pkgarch=${pkginfo[2]} - pkglicense=(${pkginfo[@]:3}) + pkglicense=("${pkginfo[@]:3}") # Should this package be skipped? if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then @@ -105,13 +105,13 @@ for repo in ${PKGREPOS[@]}; do if [ ${#newpkgs[@]} -ge 1 ]; then msg "Adding source packages for [${repo}]..." - for new_pkg in ${newpkgs[@]}; do + for new_pkg in "${newpkgs[@]}"; do msg2 "${new_pkg}" done fi if [ ${#failedpkgs[@]} -ge 1 ]; then msg "Failed to create source packages for [${repo}]..." - for failed_pkg in ${failedpkgs[@]}; do + for failed_pkg in "${failedpkgs[@]}"; do msg2 "${failed_pkg}" done fi @@ -125,7 +125,7 @@ old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-s if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old source packages..." ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' - for old_pkg in ${old_pkgs[@]}; do + for old_pkg in "${old_pkgs[@]}"; do msg2 "${old_pkg}" if ! ${SOURCE_CLEANUP_DRYRUN}; then mv_acl "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" @@ -137,7 +137,7 @@ fi old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -name "*${SRCEXT}" -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old source packages from the cleanup directory..." - for old_pkg in ${old_pkgs[@]}; do + for old_pkg in "${old_pkgs[@]}"; do msg2 "${old_pkg}" ${SOURCE_CLEANUP_DRYRUN} || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" done diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index 1432bdf..2a26e6a 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -21,7 +21,7 @@ find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort pushd "${SVNREPO}" >/dev/null -for repo in ${PKGREPOS[@]}; do +for repo in "${PKGREPOS[@]}"; do msg "Sourceballing [${repo}]" pushd $repo >/dev/null @@ -40,7 +40,7 @@ for repo in ${PKGREPOS[@]}; do unset build package url pkgdesc source md5sums depends makedepends \ optdepends license arch options check mksource - for _pkg in ${pkgname[@]}; do + for _pkg in "${pkgname[@]}"; do unset package_${_pkg} >/dev/null 2>&1 done diff --git a/db-check-nonfree b/db-check-nonfree index 5cb7f6f..6e2dc17 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -9,16 +9,16 @@ if [ $# -ge 1 ]; then fi # TODO: this might lock too much (architectures) -for repo in ${repos[@]}; do - for pkgarch in ${ARCHES[@]}; do +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do repo_lock ${repo} ${pkgarch} || exit 1 done done msg "Check nonfree in repo:" nonfree=($(cut -d: -f1 ${BLACKLIST_FILE} | sort -u)) -for repo in ${ARCHREPOS[@]}; do - for pkgarch in ${ARCHES[@]}; do +for repo in "${ARCHREPOS[@]}"; do + for pkgarch in "${ARCHES[@]}"; do msg2 "$repo $pkgarch" if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then continue @@ -27,20 +27,20 @@ for repo in ${ARCHREPOS[@]}; do unset cleanpkgs cleanpkgs=() dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) - for pkgname in ${dbpkgs[@]}; do - if in_array ${pkgname} ${nonfree[@]}; then - cleanpkgs+=(${pkgname}) + for pkgname in "${dbpkgs[@]}"; do + if in_array "${pkgname}" "${nonfree[@]}"; then + cleanpkgs+=("${pkgname}") fi done if [ ${#cleanpkgs[@]} -ge 1 ]; then - msg2 "Nonfree: ${cleanpkgs[@]}" - arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]} + msg2 "Nonfree: ${cleanpkgs[*]}" + arch_repo_remove "${repo}" "${pkgarch}" "${cleanpkgs[@]}" fi done done -for repo in ${repos[@]}; do - for pkgarch in ${ARCHES[@]}; do +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do repo_unlock ${repo} ${pkgarch} done done diff --git a/db-functions b/db-functions index efb6172..5b10e05 100644 --- a/db-functions +++ b/db-functions @@ -67,7 +67,7 @@ cleanup() { local arch trap - EXIT INT QUIT TERM - for l in ${LOCKS[@]}; do + for l in "${LOCKS[@]}"; do repo=${l%.*} arch=${l#*.} if [ -d "$TMPDIR/.repolock.$repo.$arch" ]; then @@ -256,7 +256,7 @@ getpkgfiles() { exit 1 fi - for f in ${@}; do + for f in "${@}"; do if [ ! -f "${f}" ]; then error "Package ${f} not found!" exit 1 @@ -266,7 +266,7 @@ getpkgfiles() { fi done - echo ${@} + echo "${@}" } check_pkgfile() { @@ -279,7 +279,7 @@ check_pkgfile() { local pkgarch="$(getpkgarch ${pkgfile})" [ $? -ge 1 ] && return 1 - in_array "${pkgarch}" ${ARCHES[@]} 'any' || return 1 + in_array "${pkgarch}" "${ARCHES[@]}" 'any' || return 1 if echo "${pkgfile##*/}" | grep -q "${pkgname}-${pkgver}-${pkgarch}"; then return 0 @@ -300,13 +300,13 @@ check_pkgxbs() { [ $? -ge 1 ] && return 1 local repo="${2}" - in_array "${repo}" ${PKGREPOS[@]} || return 1 + in_array "${repo}" "${PKGREPOS[@]}" || return 1 local xbsver="$(. "`xbs releasepath ${_pkgbase} ${repo} ${_pkgarch}`/PKGBUILD"; get_full_version "${_pkgname}")" [ "${xbsver}" == "${_pkgver}" ] || return 1 local xbsnames=($(. "`xbs releasepath ${_pkgbase} ${repo} ${_pkgarch}`/PKGBUILD"; echo ${pkgname[@]})) - in_array "${_pkgname}" ${xbsnames[@]} || return 1 + in_array "${_pkgname}" "${xbsnames[@]}" || return 1 return 0 } @@ -314,7 +314,7 @@ check_pkgxbs() { check_splitpkgs() { local repo="${1}" shift - local pkgfiles=(${@}) + local pkgfiles=("${@}") local pkgfile local pkgdir local xbsname @@ -322,7 +322,7 @@ check_splitpkgs() { mkdir -p "${WORKDIR}/check_splitpkgs/" pushd "${WORKDIR}/check_splitpkgs" >/dev/null - for pkgfile in ${pkgfiles[@]}; do + for pkgfile in "${pkgfiles[@]}"; do issplitpkg "${pkgfile}" || continue local _pkgbase="$(getpkgbase ${pkgfile})" msg2 "Checking %s" "$_pkgbase" @@ -332,7 +332,7 @@ check_splitpkgs() { echo "${_pkgname}" >> "${repo}/${_pkgarch}/${_pkgbase}/staging" local xbsnames=($(. "`xbs releasepath ${_pkgbase} ${repo} ${_pkgarch}`/PKGBUILD"; echo ${pkgname[@]})) - for xbsname in ${xbsnames[@]}; do + for xbsname in "${xbsnames[@]}"; do echo "${xbsname}" >> "${repo}/${_pkgarch}/${_pkgbase}/xbs" done done @@ -419,13 +419,13 @@ set_repo_permission() { arch_repo_add() { local repo=$1 local arch=$2 - local pkgs=(${@:3}) + local pkgs=("${@:3}") # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null - /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ + /usr/bin/repo-add -q "${repo}${DBEXT}" "${pkgs[@]}" >/dev/null \ || error "repo-add ${repo}${DBEXT} ${pkgs[*]}" - /usr/bin/repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ + /usr/bin/repo-add -f -q "${repo}${FILESEXT}" "${pkgs[@]}" \ || error "repo-add -f ${repo}${FILESEXT} ${pkgs[*]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" @@ -436,7 +436,7 @@ arch_repo_add() { arch_repo_remove() { local repo=$1 local arch=$2 - local pkgs=(${@:3}) + local pkgs=("${@:3}") local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" @@ -444,9 +444,9 @@ arch_repo_remove() { error "No database found at '${dbfile}'" return 1 fi - /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ + /usr/bin/repo-remove -q "${dbfile}" "${pkgs[@]}" >/dev/null \ || error "repo-remove ${dbfile} ${pkgs[*]}" - /usr/bin/repo-remove -q "${filesfile}" ${pkgs[@]} \ + /usr/bin/repo-remove -q "${filesfile}" "${pkgs[@]}" \ || error "repo-remove ${filesfile} ${pkgs[*]}" set_repo_permission "${repo}" "${arch}" diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages index 5105096..f593686 100755 --- a/db-list-unsigned-packages +++ b/db-list-unsigned-packages @@ -31,7 +31,7 @@ fi arch=$1 shift -for repo in ${PKGREPOS[@]} +for repo in "${PKGREPOS[@]}" do db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" [ -f "$db" ] && "$(dirname "$(readlink -e "$0")")/db-list-unsigned-packages.py" "$repo" "$@" < "$db" diff --git a/db-move b/db-move index 1b34404..c66b088 100755 --- a/db-move +++ b/db-move @@ -8,7 +8,7 @@ if [ $# -lt 3 ]; then exit 1 fi -args=(${@}) +args=("${@}") repo_from="${args[0]}" repo_to="${args[1]}" ftppath_from="${FTP_BASE}/${repo_from}/os/" @@ -19,14 +19,14 @@ if ! check_repo_permission $repo_to || ! check_repo_permission $repo_from; then fi # TODO: this might lock too much (architectures) -for pkgarch in ${ARCHES[@]}; do +for pkgarch in "${ARCHES[@]}"; do repo_lock ${repo_to} ${pkgarch} || exit 1 repo_lock ${repo_from} ${pkgarch} || exit 1 done # check if packages to be moved exist in xbs and ftp dir -for pkgbase in ${args[@]:2}; do - for pkgarch in ${ARCHES[@]} 'any'; do +for pkgbase in "${args[@]:2}"; do + for pkgarch in "${ARCHES[@]}" 'any'; do xbsrepo_from="$(xbs releasepath ${pkgbase} ${repo_from} ${pkgarch})" if [ -r "${xbsrepo_from}/PKGBUILD" ]; then pkgnames=($(. "${xbsrepo_from}/PKGBUILD"; echo ${pkgname[@]})) @@ -34,19 +34,18 @@ for pkgbase in ${args[@]:2}; do die "Could not read pkgname" fi - if [ "${pkgarch}" == 'any' ]; then - tarches=(${ARCHES[@]}) + tarches=("${ARCHES[@]}") else tarches=("${pkgarch}") fi - for pkgname in ${pkgnames[@]}; do + for pkgname in "${pkgnames[@]}"; do pkgver=$(. "${xbsrepo_from}/PKGBUILD"; get_full_version ${pkgname}) if [ -z "${pkgver}" ]; then die "Could not read pkgver" fi - for tarch in ${tarches[@]}; do + for tarch in "${tarches[@]}"; do getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT} >/dev/null done done @@ -60,11 +59,11 @@ msg "Moving packages from [${repo_from}] to [${repo_to}]..." declare -A add_pkgs declare -A remove_pkgs -for pkgbase in ${args[@]:2}; do +for pkgbase in "${args[@]:2}"; do # move the package in xbs arches=($(xbs move ${repo_from} ${repo_to} ${pkgbase})) # move the package in ftp - for pkgarch in ${arches[@]}; do + for pkgarch in "${arches[@]}"; do dir_to="$(xbs releasepath $pkgbase $repo_to $pkgarch)" if true; then # to add in indent level to make merging easier if [ "${pkgarch}" == 'any' ]; then @@ -74,9 +73,9 @@ for pkgbase in ${args[@]:2}; do fi pkgnames=($(. "${dir_to}/PKGBUILD"; echo ${pkgname[@]})) - for pkgname in ${pkgnames[@]}; do + for pkgname in "${pkgnames[@]}"; do pkgver=$(. "${dir_to}/PKGBUILD"; get_full_version ${pkgname}) - for tarch in ${tarches[@]}; do + for tarch in "${tarches[@]}"; do pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT}) pkgfile="${pkgpath##*/}" @@ -92,14 +91,14 @@ for pkgbase in ${args[@]:2}; do done done -for tarch in ${ARCHES[@]}; do +for tarch in "${ARCHES[@]}"; do if [ -n "${add_pkgs[${tarch}]}" ]; then arch_repo_add "${repo_to}" "${tarch}" ${add_pkgs[${tarch}]} arch_repo_remove "${repo_from}" "${tarch}" ${remove_pkgs[${tarch}]} fi done -for pkgarch in ${ARCHES[@]}; do +for pkgarch in "${ARCHES[@]}"; do repo_unlock ${repo_from} ${pkgarch} repo_unlock ${repo_to} ${pkgarch} done diff --git a/db-remove b/db-remove index 1c25e5c..33d0933 100755 --- a/db-remove +++ b/db-remove @@ -10,7 +10,7 @@ fi repo="$1" arch="$2" -pkgbases=(${@:3}) +pkgbases=("${@:3}") if ! check_repo_permission $repo; then die "You don't have permission to remove packages from ${repo}" @@ -27,7 +27,7 @@ for tarch in "${tarches[@]}"; do done remove_pkgs=() -for pkgbase in ${pkgbases[@]}; do +for pkgbase in "${pkgbases[@]}"; do msg "Removing $pkgbase from [$repo]..." path="$(xbs releasepath "$pkgbase" "$repo" "$arch")" if [ -d "$path" ]; then @@ -37,7 +37,7 @@ for pkgbase in ${pkgbases[@]}; do warning "$pkgbase not found in XBS $repo-$arch" warning "Removing only $pkgbase from the repo" warning "If it was a split package you have to remove the others yourself!" - remove_pkgs+=($pkgbase) + remove_pkgs+=("$pkgbase") fi done diff --git a/db-sync b/db-sync index e4b6966..3595876 100755 --- a/db-sync +++ b/db-sync @@ -67,8 +67,8 @@ init() { get_repos # Traverse all repo-arch pairs - for _repo in ${ARCHREPOS[@]}; do - for _arch in ${ARCHARCHES[@]}; do + for _repo in "${ARCHREPOS[@]}"; do + for _arch in "${ARCHARCHES[@]}"; do msg "Processing ${_repo}-${_arch}" db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT} @@ -165,7 +165,6 @@ init() { rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ ${FTP_BASE}/${SRCPOOL}/ done - # Cleanup unset blacklist whitelists _arch _repo repo_file diff --git a/db-update b/db-update index 8cf61fc..c4bd33b 100755 --- a/db-update +++ b/db-update @@ -16,26 +16,26 @@ fi repos=() for staging_repo in ${staging_repos[@]##*/}; do - if in_array ${staging_repo} ${PKGREPOS[@]}; then - repos+=(${staging_repo}) + if in_array "${staging_repo}" "${PKGREPOS[@]}"; then + repos+=("${staging_repo}") fi done # TODO: this might lock too much (architectures) -for repo in ${repos[@]}; do - for pkgarch in ${ARCHES[@]}; do +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do repo_lock ${repo} ${pkgarch} || exit 1 done done # check if packages are valid -for repo in ${repos[@]}; do +for repo in "${repos[@]}"; do pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXT})) if [ $? -eq 0 ]; then if [ ${#pkgs[@]} -gt 0 ] && ! check_repo_permission "${repo}"; then die "You don't have permission to update packages in ${repo}" fi - for pkg in ${pkgs[@]}; do + for pkg in "${pkgs[@]}"; do if [ -h "${pkg}" ]; then die "Package ${repo}/${pkg##*/} is a symbolic link" fi @@ -60,13 +60,13 @@ for repo in ${repos[@]}; do fi done -for repo in ${repos[@]}; do +for repo in "${repos[@]}"; do msg "Updating [${repo}]..." any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXT} 2>/dev/null)) - for pkgarch in ${ARCHES[@]}; do + for pkgarch in "${ARCHES[@]}"; do add_pkgs=() arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-${pkgarch}${PKGEXT} 2>/dev/null)) - for pkg in ${arch_pkgs[@]} ${any_pkgs[@]}; do + for pkg in "${arch_pkgs[@]}" "${any_pkgs[@]}"; do pkgfile="${pkg##*/}" msg2 "${pkgfile} (${pkgarch})" # any packages might have been moved by the previous run @@ -81,16 +81,16 @@ for repo in ${repos[@]}; do if [ -f "$FTP_BASE/${PKGPOOL}/${pkgfile}.sig" ]; then ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "$FTP_BASE/$repo/os/${pkgarch}" fi - add_pkgs[${#add_pkgs[*]}]=${pkgfile} + add_pkgs+=("${pkgfile}") done if [ ${#add_pkgs[@]} -ge 1 ]; then - arch_repo_add "${repo}" "${pkgarch}" ${add_pkgs[@]} + arch_repo_add "${repo}" "${pkgarch}" "${add_pkgs[@]}" fi done done -for repo in ${repos[@]}; do - for pkgarch in ${ARCHES[@]}; do +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do repo_unlock ${repo} ${pkgarch} done done diff --git a/get-repos b/get-repos index 5096433..b8d2ccb 100755 --- a/get-repos +++ b/get-repos @@ -25,8 +25,8 @@ trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT DBLIST=() # Repos -for _repo in ${PKGREPOS[@]}; do - for _arch in ${ARCHES[@]}; do +for _repo in "${PKGREPOS[@]}"; do + for _arch in "${ARCHES[@]}"; do DBLIST+=("http://repo.parabolagnulinux.org/${_repo}/os/${_arch}/${_repo}${FILESEXT}") done done @@ -37,7 +37,7 @@ wget --directory-prefix=${WORKDIR} \ --no-verbose \ --force-directories \ --no-host-directories \ - ${DBLIST[@]} || true + "${DBLIST[@]}" || true # Always return true, some databases are expect to be missing # Create the arches regexp arch1|arch2 diff --git a/repo-restore-to-normal b/repo-restore-to-normal index 3fe4816..063aacf 100755 --- a/repo-restore-to-normal +++ b/repo-restore-to-normal @@ -12,7 +12,7 @@ PKGREPOS=(community) # sed "s/^\(.\+-[^-]\+-[^-]\+\)-[^-]\+$/\1/")) # Traverse all repos -for _repo in ${PKGREPOS[@]}; do +for _repo in "${PKGREPOS[@]}"; do msg "Restoring [${_repo}]" # Find all pkgnames on this repo's abs @@ -27,7 +27,7 @@ for _repo in ${PKGREPOS[@]}; do >/dev/null 2>&1 # also cleanup package functions - for _pkg in ${pkgname[@]}; do + for _pkg in "${pkgname[@]}"; do unset package_${pkg} >/dev/null 2>&1 # this fills the on_abs array echo ${_pkg}-${pkgver}-${pkgrel} @@ -49,7 +49,7 @@ for _repo in ${PKGREPOS[@]}; do msg2 "Restoring the following packages:" # plain "$(echo ${restore[@]} | tr ' ' "\n")" - for _pkg in ${on_abs[@]}; do + for _pkg in "${on_abs[@]}"; do find ${CLEANUP_DESTDIR} -name "${_pkg}*" -exec cp -v '{}' ${STAGING}/${_repo} \; done -- cgit v1.2.3-2-g168b From 607b95c40e8ff5fbb452a1594d96ec6cac43df40 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 20:56:24 -0500 Subject: fix local_config:mirrorpath for the selected mirror --- local_config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local_config b/local_config index ec22d84..d329e17 100644 --- a/local_config +++ b/local_config @@ -8,7 +8,7 @@ mirror="mirrors.kernel.org" #mirror="archlinux.c3sl.ufpr.br" #mirror="mirror.us.leaseweb.net" #mirror="mirror.de.leaseweb.net" -mirrorpath="mirrors/archlinux" +mirrorpath="archlinux" # mkrepo repodir=${_paraboladir}/repo -- cgit v1.2.3-2-g168b From cad2d8b7075cb255f76e79c12c23acee94074514 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 21:02:13 -0500 Subject: repo-sanity-check: take advantage of printf --- cron-jobs/repo-sanity-check | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index 9d351df..8b0758f 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -41,8 +41,9 @@ for _repo in "${PKGREPOS[@]}"; do )) # Compares them, whatever is on repos but not on abs should be removed - remove=($(comm -13 <(echo ${on_abs[@]} | tr ' ' "\n" | sort -u) \ - <(echo ${on_repo[@]} | tr ' ' "\n" | sort -u))) + remove=($(comm -13 \ + <(printf '%s\n' "${on_abs[@]}" | sort -u) \ + <(printf '%s\n' "${on_repo[@]}" | sort -u) )) # Remove them from databases, ftpdir-cleanup will take care of the rest find ${FTP_BASE}/${_repo} -name "*.db.tar.?z" -exec \ -- cgit v1.2.3-2-g168b From 766a076891a37cebc7b448241c8c183c32f49d9e Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 18:05:41 -0500 Subject: rm repo-remove -- in Oct 2012 fauno removed repo-add, but left this The customized repo-{add,remove} did license extraction for packages. They were based on the versions from pacman 3.5.0 Here is a diff between the stock versions from 3.5.0, and the modified version that we had: --- repo-add.sh.in 2013-12-31 18:02:13.546351038 -0500 +++ repo-remove.in 2013-12-31 18:13:19.957948677 -0500 @@ -20,6 +20,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +LICENSESDIR=/home/parabolavnx/licenses + # gettext initialization export TEXTDOMAIN='pacman' export TEXTDOMAINDIR='@localedir@' @@ -309,6 +311,22 @@ fi fi + # Extracts licenses to a common license dir + msg "Extracting license" + if bsdtar -xOf ${pkgfile} .PKGINFO | grep "license" | grep "custom" ; then + if [ -d ${LICENSESDIR}/${pkgname} ]; then + rm -r ${LICENSESDIR}/${pkgname} + fi + + # Change dir to licenses, and extract them stripping the first part of the path + bsdtar -C ${LICENSESDIR}/ --include="usr/share/licenses/" \ + --strip-components 3 -xf ${pkgfile} >/dev/null 2>&1 + + if [ $? -ne 0 ]; then + warning "This package doesn't contain a license dir" + fi + fi + return 0 } # end db_write_entry @@ -328,6 +346,12 @@ rm -rf $pkgentry pkgentry=$(find_pkgentry $pkgname) done + + msg "Removing license" + if [ -d ${LICENSESDIR}/${pkgname} ]; then + rm -r ${LICENSESDIR}/${pkgname} + fi + return $notfound } # end db_remove_entry --- repo-remove | 561 ------------------------------------------------------------ 1 file changed, 561 deletions(-) delete mode 100755 repo-remove diff --git a/repo-remove b/repo-remove deleted file mode 100755 index c4bf96f..0000000 --- a/repo-remove +++ /dev/null @@ -1,561 +0,0 @@ -#!/bin/bash -# -# repo-add - add a package to a given repo database file -# repo-remove - remove a package entry from a given repo database file -# Generated from repo-add.in; do not edit by hand. -# -# Copyright (c) 2006-2008 Aaron Griffin -# Copyright (c) 2007-2008 Dan McGee -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -LICENSESDIR=/home/parabolavnx/licenses - -# gettext initialization -export TEXTDOMAIN='pacman' -export TEXTDOMAINDIR='/usr/share/locale' - -myver='3.5.0' -confdir='/home/parabolavnx/etc' - -QUIET=0 -DELTA=0 -WITHFILES=0 -REPO_DB_FILE= -LOCKFILE= -CLEAN_LOCK=0 - -# ensure we have a sane umask set -umask 0022 - -msg() { - (( QUIET )) && return - local mesg=$1; shift - printf "==> ${mesg}\n" "$@" >&1 -} - -msg2() { - (( QUIET )) && return - local mesg=$1; shift - printf " -> ${mesg}\n" "$@" >&1 -} - -warning() { - local mesg=$1; shift - printf "==> $(gettext "WARNING:") ${mesg}\n" "$@" >&2 -} - -error() { - local mesg=$1; shift - printf "==> $(gettext "ERROR:") ${mesg}\n" "$@" >&2 -} - -# print usage instructions -usage() { - printf "repo-add, repo-remove (pacman) %s\n\n" "$myver" - printf "$(gettext "Usage: repo-add [-d] [-f] [-q] ...\n")" - printf "$(gettext "Usage: repo-remove [-q] ...\n\n")" - printf "$(gettext "\ -repo-add will update a package database by reading a package file.\n\ -Multiple packages to add can be specified on the command line.\n\n")" - printf "$(gettext "\ -repo-remove will update a package database by removing the package name\n\ -specified on the command line from the given repo database. Multiple\n\ -packages to remove can be specified on the command line.\n\n")" - printf "$(gettext "\ -Use the -q/--quiet flag to minimize output to basic messages, warnings,\n\ -and errors.\n\n")" - printf "$(gettext "\ -Use the -d/--delta flag to automatically generate and add a delta file\n\ -between the old entry and the new one, if the old package file is found\n\ -next to the new one.\n\n")" - printf "$(gettext "\ -Use the -f/--files flag to update a database including file entries.\n\n")" - echo "$(gettext "Example: repo-add /path/to/repo.db.tar.gz pacman-3.0.0.pkg.tar.gz")" - echo "$(gettext "Example: repo-remove /path/to/repo.db.tar.gz kernel26")" -} - -version() { - printf "repo-add, repo-remove (pacman) %s\n\n" "$myver" - printf "$(gettext "\ -Copyright (C) 2006-2008 Aaron Griffin .\n\ -Copyright (c) 2007-2008 Dan McGee .\n\n\ -This is free software; see the source for copying conditions.\n\ -There is NO WARRANTY, to the extent permitted by law.\n")" -} - -# write a list entry -# arg1 - Entry name -# arg2 - List -# arg3 - File to write to -write_list_entry() { - if [[ -n $2 ]]; then - echo "%$1%" >>$3 - echo -e $2 >>$3 - fi -} - -find_pkgentry() -{ - local pkgname=$1 - local pkgentry - for pkgentry in $tmpdir/$pkgname*; do - name=${pkgentry##*/} - if [[ ${name%-*-*} = $pkgname ]]; then - echo $pkgentry - return 0 - fi - done - return 1 -} - -# Get the package name from the delta filename -get_delta_pkgname() { - local tmp - - tmp=${1##*/} - echo ${tmp%-*-*_to*} -} - -# write a delta entry -# arg1 - path to delta file -db_write_delta() -{ - deltafile="$1" - pkgname="$(get_delta_pkgname $deltafile)" - - pkgentry=$(find_pkgentry $pkgname) - if [[ -z $pkgentry ]]; then - error "$(gettext "No database entry for package '%s'.")" "$pkgname" - return 1 - fi - deltas="$pkgentry/deltas" - if [[ ! -f $deltas ]]; then - echo -e "%DELTAS%" >$deltas - fi - # get md5sum and compressed size of package - md5sum="$(openssl dgst -md5 "$deltafile")" - md5sum="${md5sum##* }" - csize=$(stat -L -c %s "$deltafile") - - oldfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (source)" | sed 's/.*: *//') - newfile=$(xdelta3 printhdr $deltafile | grep "XDELTA filename (output)" | sed 's/.*: *//') - - if grep -q "$oldfile.*$newfile" $deltas; then - sed -i.backup "/$oldfile.*$newfile/d" $deltas && rm -f $deltas.backup - fi - msg2 "$(gettext "Adding 'deltas' entry : %s -> %s")" "$oldfile" "$newfile" - echo ${deltafile##*/} $md5sum $csize $oldfile $newfile >> $deltas - - return 0 -} # end db_write_delta - -# remove a delta entry -# arg1 - path to delta file -db_remove_delta() -{ - deltafile="$1" - filename=${deltafile##*/} - pkgname="$(get_delta_pkgname $deltafile)" - - pkgentry=$(find_pkgentry $pkgname) - if [[ -z $pkgentry ]]; then - return 1 - fi - deltas="$pkgentry/deltas" - if [[ ! -f $deltas ]]; then - return 1 - fi - if grep -q "$filename" $deltas; then - sed -i.backup "/$filename/d" $deltas && rm -f $deltas.backup - msg2 "$(gettext "Removing existing entry '%s'...")" "$filename" - return 0 - fi - - return 1 -} # end db_remove_delta - -# write an entry to the pacman database -# arg1 - path to package -db_write_entry() -{ - # blank out all variables - local pkgfile="$1" - local pkgname pkgver pkgdesc csize size md5sum url arch builddate packager \ - _groups _licenses _replaces _depends _conflicts _provides _optdepends - - local OLDIFS="$IFS" - # IFS (field separator) is only the newline character - IFS=" -" - - # read info from the zipped package - local line var val - for line in $(bsdtar -xOqf "$pkgfile" .PKGINFO | - grep -v '^#' | sed 's|\(\w*\)\s*=\s*\(.*\)|\1 \2|'); do - # bash awesomeness here- var is always one word, val is everything else - var=${line%% *} - val=${line#* } - declare $var="$val" - case "$var" in - group) _groups="$_groups$group\n" ;; - license) _licenses="$_licenses$license\n" ;; - replaces) _replaces="$_replaces$replaces\n" ;; - depend) _depends="$_depends$depend\n" ;; - conflict) _conflicts="$_conflicts$conflict\n" ;; - provides) _provides="$_provides$provides\n" ;; - optdepend) _optdepends="$_optdepends$optdepend\n" ;; - esac - done - - IFS=$OLDIFS - - # get md5sum and compressed size of package - md5sum="$(openssl dgst -md5 "$pkgfile")" - md5sum="${md5sum##* }" - csize=$(stat -L -c %s "$pkgfile") - - # ensure $pkgname and $pkgver variables were found - if [[ -z $pkgname || -z $pkgver ]]; then - error "$(gettext "Invalid package file '%s'.")" "$pkgfile" - return 1 - fi - - pushd "$tmpdir" >/dev/null - if [[ -d $pkgname-$pkgver ]]; then - warning "$(gettext "An entry for '%s' already existed")" "$pkgname-$pkgver" - else - if (( DELTA )); then - pkgentry=$(find_pkgentry $pkgname) - if [[ -n $pkgentry ]]; then - local oldfilename=$(grep -A1 FILENAME $pkgentry/desc | tail -n1) - local oldfile="$(dirname $1)/$oldfilename" - fi - fi - fi - - # remove an existing entry if it exists, ignore failures - db_remove_entry "$pkgname" - - # create package directory - mkdir "$pkgname-$pkgver" - pushd "$pkgname-$pkgver" >/dev/null - - # restore an eventual deltas file - [[ -f ../$pkgname.deltas ]] && mv "../$pkgname.deltas" deltas - - # create desc entry - msg2 "$(gettext "Creating '%s' db entry...")" 'desc' - echo -e "%FILENAME%\n$(basename "$1")\n" >>desc - echo -e "%NAME%\n$pkgname\n" >>desc - [[ -n $pkgbase ]] && echo -e "%BASE%\n$pkgbase\n" >>desc - echo -e "%VERSION%\n$pkgver\n" >>desc - [[ -n $pkgdesc ]] && echo -e "%DESC%\n$pkgdesc\n" >>desc - write_list_entry "GROUPS" "$_groups" "desc" - [[ -n $csize ]] && echo -e "%CSIZE%\n$csize\n" >>desc - [[ -n $size ]] && echo -e "%ISIZE%\n$size\n" >>desc - - # compute checksums - msg2 "$(gettext "Computing md5 checksums...")" - echo -e "%MD5SUM%\n$md5sum\n" >>desc - - [[ -n $url ]] && echo -e "%URL%\n$url\n" >>desc - write_list_entry "LICENSE" "$_licenses" "desc" - [[ -n $arch ]] && echo -e "%ARCH%\n$arch\n" >>desc - [[ -n $builddate ]] && echo -e "%BUILDDATE%\n$builddate\n" >>desc - [[ -n $packager ]] && echo -e "%PACKAGER%\n$packager\n" >>desc - write_list_entry "REPLACES" "$_replaces" "desc" - - # create depends entry - msg2 "$(gettext "Creating '%s' db entry...")" 'depends' - # create the file even if it will remain empty - touch "depends" - write_list_entry "DEPENDS" "$_depends" "depends" - write_list_entry "CONFLICTS" "$_conflicts" "depends" - write_list_entry "PROVIDES" "$_provides" "depends" - write_list_entry "OPTDEPENDS" "$_optdepends" "depends" - - popd >/dev/null - popd >/dev/null - - # create files file if wanted - if (( WITHFILES )); then - msg2 "$(gettext "Creating '%s' db entry...")" 'files' - local files_path="$tmpdir/$pkgname-$pkgver/files" - echo "%FILES%" >$files_path - bsdtar --exclude='.*' -tf "$pkgfile" >>$files_path - fi - - # create a delta file - if (( DELTA )); then - if [[ -n $oldfilename ]]; then - if [[ -f $oldfile ]]; then - delta=$(pkgdelta -q $oldfile $1) - if [[ -f $delta ]]; then - db_write_delta $delta - fi - else - warning "$(gettext "Old package file not found: %s")" "$oldfilename" - fi - fi - fi - - # Extracts licenses to a common license dir - msg "Extracting license" - if bsdtar -xOf ${pkgfile} .PKGINFO | grep "license" | grep "custom" ; then - if [ -d ${LICENSESDIR}/${pkgname} ]; then - rm -r ${LICENSESDIR}/${pkgname} - fi - - # Change dir to licenses, and extract them stripping the first part of the path - bsdtar -C ${LICENSESDIR}/ --include="usr/share/licenses/" \ - --strip-components 3 -xf ${pkgfile} >/dev/null 2>&1 - - if [ $? -ne 0 ]; then - warning "This package doesn't contain a license dir" - fi - fi - - return 0 -} # end db_write_entry - -# remove existing entries from the DB -# arg1 - package name -db_remove_entry() { - local pkgname=$1 - local notfound=1 - local pkgentry=$(find_pkgentry $pkgname) - while [[ -n $pkgentry ]]; do - notfound=0 - if [[ -f $pkgentry/deltas ]]; then - mv "$pkgentry/deltas" "$tmpdir/$pkgname.deltas" - fi - msg2 "$(gettext "Removing existing entry '%s'...")" \ - "$(basename $pkgentry)" - rm -rf $pkgentry - pkgentry=$(find_pkgentry $pkgname) - done - - msg "Removing license" - if [ -d ${LICENSESDIR}/${pkgname} ]; then - rm -r ${LICENSESDIR}/${pkgname} - fi - - return $notfound -} # end db_remove_entry - -check_repo_db() -{ - # check lock file - if ( set -o noclobber; echo "$$" > "$LOCKFILE") 2> /dev/null; then - CLEAN_LOCK=1 - else - error "$(gettext "Failed to acquire lockfile: %s.")" "$LOCKFILE" - [[ -f $LOCKFILE ]] && error "$(gettext "Held by process %s")" "$(cat $LOCKFILE)" - exit 1 - fi - - if [[ -f $REPO_DB_FILE ]]; then - # there are two situations we can have here- a DB with some entries, - # or a DB with no contents at all. - if ! bsdtar -tqf "$REPO_DB_FILE" '*/desc' >/dev/null 2>&1; then - # check empty case - if [[ -n $(bsdtar -tqf "$REPO_DB_FILE" '*' 2>/dev/null) ]]; then - error "$(gettext "Repository file '%s' is not a proper pacman database.")" "$REPO_DB_FILE" - exit 1 - fi - fi - msg "$(gettext "Extracting database to a temporary location...")" - bsdtar -xf "$REPO_DB_FILE" -C "$tmpdir" - else - case "$cmd" in - repo-remove) - error "$(gettext "Repository file '%s' was not found.")" "$REPO_DB_FILE" - exit 1 - ;; - repo-add) - # check if the file can be created (write permission, directory existence, etc) - if ! touch "$REPO_DB_FILE"; then - error "$(gettext "Repository file '%s' could not be created.")" "$REPO_DB_FILE" - exit 1 - fi - rm -f "$REPO_DB_FILE" - ;; - esac - fi -} - -add() -{ - if [[ ! -f $1 ]]; then - error "$(gettext "File '%s' not found.")" "$1" - return 1 - fi - - if [[ ${1##*.} == "delta" ]]; then - deltafile=$1 - msg "$(gettext "Adding delta '%s'")" "$deltafile" - if ! type xdelta3 &>/dev/null; then - error "$(gettext "Cannot find the xdelta3 binary! Is xdelta3 installed?")" - exit 1 - fi - if db_write_delta "$deltafile"; then - return 0 - else - return 1 - fi - fi - - pkgfile=$1 - if ! bsdtar -tqf "$pkgfile" .PKGINFO >/dev/null 2>&1; then - error "$(gettext "'%s' is not a package file, skipping")" "$pkgfile" - return 1 - fi - - msg "$(gettext "Adding package '%s'")" "$pkgfile" - - db_write_entry "$pkgfile" -} - -remove() -{ - if [[ ${1##*.} == "delta" ]]; then - deltafile=$1 - msg "$(gettext "Searching for delta '%s'...")" "$deltafile" - if db_remove_delta "$deltafile"; then - return 0 - else - error "$(gettext "Delta matching '%s' not found.")" "$deltafile" - return 1 - fi - fi - - pkgname=$1 - msg "$(gettext "Searching for package '%s'...")" "$pkgname" - - if db_remove_entry "$pkgname"; then - rm -f "$tmpdir/$pkgname.deltas" - return 0 - else - error "$(gettext "Package matching '%s' not found.")" "$pkgname" - return 1 - fi -} - -trap_exit() -{ - echo - error "$@" - exit 1 -} - -clean_up() { - local exit_code=$? - - [[ -d $tmpdir ]] && rm -rf "$tmpdir" - (( CLEAN_LOCK )) && [[ -f $LOCKFILE ]] && rm -f "$LOCKFILE" - - exit $exit_code -} - -# PROGRAM START - -# determine whether we have gettext; make it a no-op if we do not -if ! type gettext &>/dev/null; then - gettext() { - echo "$@" - } -fi - -case "$1" in - -h|--help) usage; exit 0;; - -V|--version) version; exit 0;; -esac - -# figure out what program we are -cmd="$(basename $0)" -if [[ $cmd != "repo-add" && $cmd != "repo-remove" ]]; then - error "$(gettext "Invalid command name '%s' specified.")" "$cmd" - exit 1 -fi - -tmpdir=$(mktemp -d /tmp/repo-tools.XXXXXXXXXX) || (\ - error "$(gettext "Cannot create temp directory for database building.")"; \ - exit 1) - -trap 'clean_up' EXIT -trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT -trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT -trap 'trap_exit "$(gettext "An unknown error has occured. Exiting...")"' ERR - -success=0 -# parse arguments -for arg in "$@"; do - case "$arg" in - -q|--quiet) QUIET=1;; - -d|--delta) DELTA=1;; - -f|--files) WITHFILES=1;; - *) - if [[ -z $REPO_DB_FILE ]]; then - REPO_DB_FILE="$arg" - LOCKFILE="$REPO_DB_FILE.lck" - check_repo_db - else - case "$cmd" in - repo-add) add $arg && success=1 ;; - repo-remove) remove $arg && success=1 ;; - esac - fi - ;; - esac -done - -# if at least one operation was a success, re-zip database -if (( success )); then - msg "$(gettext "Creating updated database file '%s'")" "$REPO_DB_FILE" - - case "$REPO_DB_FILE" in - *tar.gz) TAR_OPT="z" ;; - *tar.bz2) TAR_OPT="j" ;; - *tar.xz) TAR_OPT="J" ;; - *) warning "$(gettext "'%s' does not have a valid archive extension.")" \ - "$REPO_DB_FILE" ;; - esac - - filename=$(basename "$REPO_DB_FILE") - - pushd "$tmpdir" >/dev/null - if [[ -n $(ls) ]]; then - bsdtar -c${TAR_OPT}f "$filename" * - else - # we have no packages remaining? zip up some emptyness - warning "$(gettext "No packages remain, creating empty database.")" - bsdtar -c${TAR_OPT}f "$filename" -T /dev/null - fi - popd >/dev/null - - [[ -f $REPO_DB_FILE ]] && mv -f "$REPO_DB_FILE" "${REPO_DB_FILE}.old" - [[ -f $tmpdir/$filename ]] && mv "$tmpdir/$filename" "$REPO_DB_FILE" - dblink="${REPO_DB_FILE%.tar.*}" - target=${REPO_DB_FILE##*/} - ln -sf "$target" "$dblink" 2>/dev/null || \ - ln -f "$target" "$dblink" 2>/dev/null || \ - cp "$REPO_DB_FILE" "$dblink" -else - msg "$(gettext "No packages modified, nothing to do.")" - exit 1 -fi - -exit 0 -# vim: set ts=2 sw=2 noet: -- cgit v1.2.3-2-g168b From 3537841a9287ea1c7871545ffebb855561f7c1e0 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 21:17:35 -0500 Subject: misc touch up - TODO: add trailing newline - config: add text editor hint - config.orig: remove - getrepos: quote, use -- - db-functions: hardcode some paths, remove needless nullglob --- TODO | 2 +- config | 2 +- config.orig | 54 ------------------------------------------------------ createrepos | 2 +- db-functions | 21 ++++++++------------- 5 files changed, 11 insertions(+), 70 deletions(-) delete mode 100644 config.orig diff --git a/TODO b/TODO index 3219b1c..9dd4b52 100644 --- a/TODO +++ b/TODO @@ -7,4 +7,4 @@ * Fix db-move - - Make it use abslibre \ No newline at end of file + - Make it use abslibre diff --git a/config b/config index 8b8117f..0152b69 100644 --- a/config +++ b/config @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash # as a hint to text editors FTP_BASE="/srv/http/repo/public" ARCH_BASE="/srv/http/repo/public" SVNREPO="/var/abs" diff --git a/config.orig b/config.orig deleted file mode 100644 index a32f82f..0000000 --- a/config.orig +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -<<<<<<< HEAD -FTP_BASE="/srv/http/repo/public" -ARCH_BASE="/srv/http/repo/public" -SVNREPO="/srv/http/repo/abslibre" -======= -FTP_BASE="/srv/http/repo/public/temprepo" -ARCH_BASE="/srv/http/repo/public/temprepo" -SVNREPO="/var/abs" ->>>>>>> 801ea2c927ace5ee892209dd8e3c1044e1b3842e - -# Repos from Arch -ARCHREPOS=('core' 'testing') #'extra' 'community' 'testing' 'multilib') -# Official Parabola repos -OURREPOS=('libre' 'libre-testing') -# User repos -USERREPOS=('~fauno' '~smv' '~xihh' '~mtjm' '~brendan') -# Community project repos -PROJREPOS=('social' 'elementary' 'kernels' 'radio' 'security' 'sugar') -PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) -PKGPOOL='pool/packages' -SRCPOOL='sources/packages' - -CLEANUP_DESTDIR="$FTP_BASE/old/packages" -CLEANUP_DRYRUN=true -# Time in days to keep moved packages -CLEANUP_KEEP=30 - -SOURCE_CLEANUP_DESTDIR="$FTP_BASE/old/sources" -SOURCE_CLEANUP_DRYRUN=true -# Time in days to keep moved sourcepackages -SOURCE_CLEANUP_KEEP=30 - -REQUIRE_SIGNATURE=true - -LOCK_DELAY=10 -LOCK_TIMEOUT=300 - -STAGING="$FTP_BASE/staging" -TMPDIR="/tmp" -ARCHARCHES=(i686 x86_64) -OURARCHES=(mips64el) -ARCHES=(${ARCHARCHES[@]} ${OURARCHES[@]}) -DBEXT=".db.tar.gz" -FILESEXT=".files.tar.gz" -PKGEXT=".pkg.tar.?z" -SRCEXT=".src.tar.gz" - -<<<<<<< HEAD -MAKEPKGCONF="~/.makepkg.conf" -======= -MAKEPKGCONF="/etc/makepkg.conf" ->>>>>>> 801ea2c927ace5ee892209dd8e3c1044e1b3842e -BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" diff --git a/createrepos b/createrepos index 4ee057b..926c158 100755 --- a/createrepos +++ b/createrepos @@ -3,6 +3,6 @@ source $(dirname $0)/config -mkdir -p ${FTP_BASE}/{${PKGPOOL},${SRCPOOL}} ${ARCH_BASE} ${CLEANUP_DESTDIR} ${SOURCE_CLEANUP_DESTDIR} ${STAGING} +mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${ARCH_BASE}" "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" $(dirname $0)/create-repo ${PKGREPOS[@]} diff --git a/db-functions b/db-functions index 4c247a7..458a370 100644 --- a/db-functions +++ b/db-functions @@ -87,7 +87,7 @@ get_full_version() { script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.$(basename $0)" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - local _owner="$(stat -c %U $LOCKDIR)" + local _owner="$(/usr/bin/stat -c %U $LOCKDIR)" error "Script $(basename $0) is already locked by $_owner." exit 1 else @@ -178,7 +178,7 @@ repo_lock () { _count=0 while [ $_count -le $_trial ] || $_lockblock ; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - _owner="$(stat -c %U $LOCKDIR)" + _owner="$(/usr/bin/stat -c %U $LOCKDIR)" warning "Repo [${1}] (${2}) is already locked by $_owner. " msg2 "Retrying in $LOCK_DELAY seconds..." else @@ -211,7 +211,7 @@ repo_unlock () { #repo_unlock _grep_pkginfo() { local _ret - _ret="$(bsdtar -xOqf "$1" .PKGINFO | grep -m 1 "^${2} = ")" + _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | grep -m 1 "^${2} = ")" echo "${_ret#${2} = }" } @@ -293,9 +293,6 @@ getpkgfile() { } getpkgfiles() { -# Ignore anything that doesn't glob to PKGEXT - shopt -s nullglob - local f if [ ! -z "$(echo ${@%\.*} | sed "s/ /\n/g" | sort | uniq -D)" ]; then error 'Duplicate packages found!' @@ -313,8 +310,6 @@ getpkgfiles() { done echo ${@} - - shopt -u nullglob } check_pkgfile() { @@ -485,7 +480,7 @@ set_repo_permission() { local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" if [ -w "${dbfile}" ]; then - local group=$(stat --printf='%G' "$(dirname "${dbfile}")") + local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") chgrp $group "${dbfile}" || error "Could not change group of ${dbfile} to $group" chgrp $group "${filesfile}" || error "Could not change group of ${filesfile} to $group" chmod g+w "${dbfile}" || error "Could not set write permission for group $group to ${dbfile}" @@ -502,9 +497,9 @@ arch_repo_add() { # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null - repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ + /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" - repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ + /usr/bin/repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ || error "repo-add -f ${repo}${FILESEXT} ${pkgs[@]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" @@ -521,9 +516,9 @@ arch_repo_remove() { error "No database found at '${dbfile}'" return 1 fi - repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ + /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ || error "repo-remove ${dbfile} ${pkgs[@]}" - repo-remove -q "${filesfile}" ${pkgs[@]} \ + /usr/bin/repo-remove -q "${filesfile}" ${pkgs[@]} \ || error "repo-remove ${filesfile} ${pkgs[@]}" set_repo_permission "${repo}" "${arch}" } -- cgit v1.2.3-2-g168b From 1bcc49058febc7e2723c455879da6ff3b9ca6576 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 21:18:36 -0500 Subject: Use `mktemp -t` to respect $TMPDIR --- db-functions | 2 +- get-repos | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/db-functions b/db-functions index 458a370..f56ca19 100644 --- a/db-functions +++ b/db-functions @@ -16,7 +16,7 @@ restore_umask () { } # set up general environment -WORKDIR=$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX) +WORKDIR=$(mktemp -dt "$(basename $0).XXXXXXXXXX") LOCKS=() # check if messages are to be printed using color diff --git a/get-repos b/get-repos index bfc08ff..796051c 100755 --- a/get-repos +++ b/get-repos @@ -20,7 +20,7 @@ trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR -TMPDIR="$(mktemp -d /tmp/$(basename $0).XXXX)" +TMPDIR="$(mktemp -dt "$(basename $0).XXXX")" DBLIST=() # Repos -- cgit v1.2.3-2-g168b From 0811dea8b9c695ed9e67b22d389142956bd1cdd8 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 17:40:26 -0500 Subject: Fix quoting on arrays. --- config | 2 +- createrepos | 2 +- db-functions | 22 +++++++++++----------- db-move | 2 +- db-remove | 14 +++++++------- db-update | 16 ++++++++-------- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/config b/config index 0152b69..9d72ebd 100644 --- a/config +++ b/config @@ -12,7 +12,7 @@ USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jor # Community project repos PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'java-ugly' 'nonprism') # Remote repos -PKGREPOS=(${ARCHREPOS[@]} ${OURREPOS[@]} ${USERREPOS[@]} ${PROJREPOS[@]}) +PKGREPOS=("${ARCHREPOS[@]}" "${OURREPOS[@]}" "${USERREPOS[@]}" "${PROJREPOS[@]}") PKGPOOL='pool/packages' SRCPOOL='sources/packages' diff --git a/createrepos b/createrepos index 926c158..1840c83 100755 --- a/createrepos +++ b/createrepos @@ -5,4 +5,4 @@ source $(dirname $0)/config mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${ARCH_BASE}" "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" -$(dirname $0)/create-repo ${PKGREPOS[@]} +$(dirname $0)/create-repo "${PKGREPOS[@]}" diff --git a/db-functions b/db-functions index f56ca19..59e9c29 100644 --- a/db-functions +++ b/db-functions @@ -445,8 +445,8 @@ check_pkgrepos() { #usage: chk_license ${license[@]}" chk_license() { local l - for l in ${@}; do - in_array ${l} ${ALLOWED_LICENSES[@]} && return 0 + for l in "${@}"; do + in_array "${l}" "${ALLOWED_LICENSES[@]}" && return 0 done return 1 @@ -458,7 +458,7 @@ check_repo_permission() { [ ${#PKGREPOS[@]} -eq 0 ] && return 1 [ -z "${PKGPOOL}" ] && return 1 - in_array "${repo}" ${PKGREPOS[@]} || return 1 + in_array "${repo}" "${PKGREPOS[@]}" || return 1 [ -w "$FTP_BASE/${PKGPOOL}" ] || return 1 @@ -497,10 +497,10 @@ arch_repo_add() { # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null - /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ - || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" - /usr/bin/repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ - || error "repo-add -f ${repo}${FILESEXT} ${pkgs[@]}" + /usr/bin/repo-add -q "${repo}${DBEXT}" "${pkgs[@]}" >/dev/null \ + || error "repo-add ${repo}${DBEXT} ${pkgs[*]}" + /usr/bin/repo-add -f -q "${repo}${FILESEXT}" "${pkgs[@]}" \ + || error "repo-add -f ${repo}${FILESEXT} ${pkgs[*]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" } @@ -516,9 +516,9 @@ arch_repo_remove() { error "No database found at '${dbfile}'" return 1 fi - /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ - || error "repo-remove ${dbfile} ${pkgs[@]}" - /usr/bin/repo-remove -q "${filesfile}" ${pkgs[@]} \ - || error "repo-remove ${filesfile} ${pkgs[@]}" + /usr/bin/repo-remove -q "${dbfile}" "${pkgs[@]}" >/dev/null \ + || error "repo-remove ${dbfile} ${pkgs[*]}" + /usr/bin/repo-remove -q "${filesfile}" "${pkgs[@]}" \ + || error "repo-remove ${filesfile} ${pkgs[*]}" set_repo_permission "${repo}" "${arch}" } diff --git a/db-move b/db-move index 010d941..69f24ac 100755 --- a/db-move +++ b/db-move @@ -66,7 +66,7 @@ for pkgbase in ${args[@]:2}; do if [ -f "${svnrepo_from}/PKGBUILD" ]; then if [ "${pkgarch}" == 'any' ]; then - tarches=(${ARCHES[@]}) + tarches=("${ARCHES[@]}") else tarches=("${pkgarch}") fi diff --git a/db-remove b/db-remove index f0785e5..255aa32 100755 --- a/db-remove +++ b/db-remove @@ -10,24 +10,24 @@ fi repo="$1" arch="$2" -pkgbases=(${@:3}) +pkgbases=("${@:3}") if ! check_repo_permission $repo; then die "You don't have permission to remove packages from ${repo}" fi if [ "$arch" == "any" ]; then - tarches=(${ARCHES[@]}) + tarches=("${ARCHES[@]}") else tarches=("$arch") fi -for tarch in ${tarches[@]}; do +for tarch in "${tarches[@]}"; do repo_lock $repo $tarch || exit 1 done remove_pkgs=() -for pkgbase in ${pkgbases[@]}; do +for pkgbase in "${pkgbases[@]}"; do msg "Removing $pkgbase from [$repo]..." if [ -d "${SVNREPO}/$repo/$pkgbase" ]; then @@ -36,11 +36,11 @@ for pkgbase in ${pkgbases[@]}; do warning "$pkgbase not found in ABS(libre)" warning "Removing only $pkgbase from the repo" warning "If it was a split package you have to remove the others yourself!" - remove_pkgs[${#remove_pkgs[*]}]=$pkgbase + remove_pkgs+=("$pkgbase") fi done -for tarch in ${tarches[@]}; do - arch_repo_remove "${repo}" "${tarch}" ${remove_pkgs[@]} +for tarch in "${tarches[@]}"; do + arch_repo_remove "${repo}" "${tarch}" "${remove_pkgs[@]}" repo_unlock $repo $tarch done diff --git a/db-update b/db-update index 1fddb8a..cdde63b 100755 --- a/db-update +++ b/db-update @@ -15,14 +15,14 @@ if [ $? -ge 1 ]; then fi # TODO: this might lock too much (architectures) -for repo in ${repos[@]}; do - for pkgarch in ${ARCHES[@]}; do +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do repo_lock ${repo} ${pkgarch} || exit 1 done done # check if packages are valid -for repo in ${repos[@]}; do +for repo in "${repos[@]}"; do pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXT})) if [ $? -eq 0 ]; then if [ ${#pkgs[@]} -gt 0 ] && ! check_repo_permission "${repo}"; then @@ -48,7 +48,7 @@ for repo in ${repos[@]}; do fi done -for repo in ${repos[@]}; do +for repo in "${repos[@]}"; do msg "Updating [${repo}]..." any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXT} 2>/dev/null)) for pkgarch in ${ARCHES[@]}; do @@ -69,16 +69,16 @@ for repo in ${repos[@]}; do if [ -f "$FTP_BASE/${PKGPOOL}/${pkgfile}.sig" ]; then ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "$FTP_BASE/$repo/os/${pkgarch}" fi - add_pkgs[${#add_pkgs[*]}]=${pkgfile} + add_pkgs+=("${pkgfile}") done if [ ${#add_pkgs[@]} -ge 1 ]; then - arch_repo_add "${repo}" "${pkgarch}" ${add_pkgs[@]} + arch_repo_add "${repo}" "${pkgarch}" "${add_pkgs[@]}" fi done done -for repo in ${repos[@]}; do - for pkgarch in ${ARCHES[@]}; do +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do repo_unlock ${repo} ${pkgarch} done done -- cgit v1.2.3-2-g168b From 1d08899c3ac82047fe614a7694e5a7235f808cfd Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 14:52:20 -0500 Subject: Remove extra local_config loads. Some files were loading local_config even if they did not use any settings from it. --- any-to-ours | 1 - db-cleanup | 1 - get-repos | 1 - 3 files changed, 3 deletions(-) diff --git a/any-to-ours b/any-to-ours index a1d6686..b927b82 100755 --- a/any-to-ours +++ b/any-to-ours @@ -8,7 +8,6 @@ trap_exit() { } source $(dirname $0)/config -source $(dirname $0)/local_config source $(dirname $0)/libremessages # From makepkg diff --git a/db-cleanup b/db-cleanup index 841800b..abad390 100755 --- a/db-cleanup +++ b/db-cleanup @@ -16,7 +16,6 @@ trap_exit() { } source $(dirname $0)/config -source $(dirname $0)/local_config source $(dirname $0)/libremessages # From makepkg diff --git a/get-repos b/get-repos index 796051c..92db9a2 100755 --- a/get-repos +++ b/get-repos @@ -10,7 +10,6 @@ trap_exit() { } source $(dirname $0)/config -source $(dirname $0)/local_config source $(dirname $0)/libremessages # From makepkg -- cgit v1.2.3-2-g168b From b6e8ebd66d22abf5439485985a7851e768c71e8a Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 21:04:43 -0500 Subject: Be very careful about using $0. --- any-to-ours | 4 ++-- create-repo | 8 ++++---- createrepos | 4 ++-- cron-jobs/ftpdir-cleanup | 4 ++-- cron-jobs/integrity-check | 4 ++-- cron-jobs/repo-sanity-check | 4 ++-- cron-jobs/sourceballs | 2 +- cron-jobs/sourceballs2 | 2 +- cron-jobs/update-abs-tarballs | 2 +- cron-jobs/update-web-db | 6 +++--- db-check-nonfree | 6 +++--- db-cleanup | 4 ++-- db-functions | 14 +++++++------- db-list-unsigned-packages | 8 ++++---- db-move | 6 +++--- db-remove | 6 +++--- db-repo-add | 6 +++--- db-repo-remove | 6 +++--- db-sync | 16 ++++++++-------- db-update | 6 +++--- get-repos | 6 +++--- migrate-repo | 2 +- mkrepo | 4 ++-- repo-restore-to-normal | 4 ++-- testing2x | 8 ++++---- yf-update | 12 ++++++------ 26 files changed, 77 insertions(+), 77 deletions(-) diff --git a/any-to-ours b/any-to-ours index b927b82..a1697c7 100755 --- a/any-to-ours +++ b/any-to-ours @@ -7,8 +7,8 @@ trap_exit() { exit 1 } -source $(dirname $0)/config -source $(dirname $0)/libremessages +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg set -E diff --git a/create-repo b/create-repo index 58842c3..24b890d 100755 --- a/create-repo +++ b/create-repo @@ -1,11 +1,11 @@ #!/bin/bash # Creates repository structure -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" if [ $# -eq 0 ]; then - msg "Usage: $0 repo1 [repo2 ... repoX]" + msg "Usage: ${0##*/} repo1 [repo2 ... repoX]" exit 1 fi @@ -21,4 +21,4 @@ for _repo in $@; do done done -msg "Don't forget to add them to the PKGREPOS array on $(dirname $0)/config" +msg "Don't forget to add them to the PKGREPOS array on %s/config" "$(dirname "$(readlink -e "$0")")" diff --git a/createrepos b/createrepos index 1840c83..a8f93e8 100755 --- a/createrepos +++ b/createrepos @@ -1,8 +1,8 @@ #!/bin/bash # Creates the repo structure defined in config -source $(dirname $0)/config +source "$(dirname "$(readlink -e "$0")")/config" mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${ARCH_BASE}" "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" -$(dirname $0)/create-repo "${PKGREPOS[@]}" +"$(dirname "$(readlink -e "$0")")/create-repo" "${PKGREPOS[@]}" diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 83e6e17..8d691b5 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" +. "$(dirname "$(readlink -e "$0")")/../config" clean_pkg() { local pkg diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index d4f9694..05a56a5 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -1,6 +1,6 @@ #!/bin/bash -dirname="$(dirname $0)" +dirname="$(dirname "$(readlink -e "$0")")" . "${dirname}/../db-functions" . "${dirname}/../config" @@ -8,7 +8,7 @@ dirname="$(dirname $0)" script_lock if [ $# -ne 1 ]; then - die "usage: $(basename $0) " + die "usage: ${0##*/} " fi mailto=$1 diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index 1ba90a6..2aa7892 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -1,8 +1,8 @@ #!/bin/bash # Solves issue165 -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" +. "$(dirname "$(readlink -e "$0")")/../config" # Traverse all repos for _repo in ${PKGREPOS[@]}; do diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index ee074bd..73d8432 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -1,6 +1,6 @@ #!/bin/bash -dirname="$(dirname $(readlink -e $0))" +dirname="$(dirname "$(readlink -e "$0")")" . "${dirname}/../db-functions" . "${dirname}/../config" pushd "${WORKDIR}" >/dev/null diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index 5644268..eb46579 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -4,7 +4,7 @@ # Makepkg --allsource every package # Remove the old sourceballs -dirname="$(dirname $(readlink -e $0))" +dirname="$(dirname "$(readlink -e "$0")")" . "${dirname}/../db-functions" . "${dirname}/../config" . "${MAKEPKGCONF}" diff --git a/cron-jobs/update-abs-tarballs b/cron-jobs/update-abs-tarballs index 824ac34..901cc4b 100755 --- a/cron-jobs/update-abs-tarballs +++ b/cron-jobs/update-abs-tarballs @@ -1,6 +1,6 @@ #!/bin/bash -. "$(dirname $0)/../config" +. "$(dirname "$(readlink -e "$0")")/../config" rsync -av --exclude=staging/ parabolagnulinux.org::abstar/ ${FTP_BASE}/ diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 6ced4c1..825eea6 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" +. "$(dirname "$(readlink -e "$0")")/../config" # setup paths SPATH="/srv/http/archweb" @@ -14,7 +14,7 @@ REPOS=('community-testing' 'multilib-testing' 'multilib' 'community' 'extra' 'te LOGOUT="/tmp/archweb_update.log" # figure out what operation to perform -cmd="$(basename $0)" +cmd="${0##*/}" if [[ $cmd != "update-web-db" && $cmd != "update-web-files-db" ]]; then die "Invalid command name '$cmd' specified!" fi diff --git a/db-check-nonfree b/db-check-nonfree index ecad3b9..661daa6 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -1,10 +1,10 @@ #!/bin/bash -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" if [ $# -ge 1 ]; then - warning "Calling $(basename $0) with a specific repository is not supported" + warning "Calling ${0##*/} with a specific repository is not supported" exit 1 fi diff --git a/db-cleanup b/db-cleanup index abad390..57ef36e 100755 --- a/db-cleanup +++ b/db-cleanup @@ -15,8 +15,8 @@ trap_exit() { exit 1 } -source $(dirname $0)/config -source $(dirname $0)/libremessages +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg set -E diff --git a/db-functions b/db-functions index 59e9c29..26b23e4 100644 --- a/db-functions +++ b/db-functions @@ -16,7 +16,7 @@ restore_umask () { } # set up general environment -WORKDIR=$(mktemp -dt "$(basename $0).XXXXXXXXXX") +WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") LOCKS=() # check if messages are to be printed using color @@ -85,10 +85,10 @@ get_full_version() { } script_lock() { - local LOCKDIR="$TMPDIR/.scriptlock.$(basename $0)" + local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then local _owner="$(/usr/bin/stat -c %U $LOCKDIR)" - error "Script $(basename $0) is already locked by $_owner." + error "Script ${0##*/} is already locked by $_owner." exit 1 else set_umask @@ -97,9 +97,9 @@ script_lock() { } script_unlock() { - local LOCKDIR="$TMPDIR/.scriptlock.$(basename $0)" + local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" if [ ! -d "$LOCKDIR" ]; then - warning "Script $(basename $0) was not locked!" + warning "Script ${0##*/} was not locked!" restore_umask return 1 else @@ -123,8 +123,8 @@ cleanup() { repo_unlock $repo $arch fi done - if [ -d "$TMPDIR/.scriptlock.$(basename $0)" ]; then - msg "Removing left over lock from $(basename $0)" + if [ -d "$TMPDIR/.scriptlock.${0##*/}" ]; then + msg "Removing left over lock from ${0##*/}" script_unlock fi rm -rf "$WORKDIR" diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages index 3b5a5bd..985d1c0 100755 --- a/db-list-unsigned-packages +++ b/db-list-unsigned-packages @@ -20,11 +20,11 @@ set -e # unsigned packages available for architecture $1 and specified for # architecture $2 (usually $1 or any, default is to list all). -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" if [ $# -lt 1 ]; then - msg "usage: $(basename $0) " + msg "usage: ${0##*/} " exit 1 fi @@ -34,5 +34,5 @@ shift for repo in ${PKGREPOS[@]} do db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" - [ -f "$db" ] && "$(dirname $0)/db-list-unsigned-packages.py" "$repo" "$@" < "$db" + [ -f "$db" ] && "$(dirname "$(readlink -e "$0")")/db-list-unsigned-packages.py" "$repo" "$@" < "$db" done diff --git a/db-move b/db-move index 69f24ac..a3f1532 100755 --- a/db-move +++ b/db-move @@ -1,10 +1,10 @@ #!/bin/bash -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" if [ $# -lt 3 ]; then - msg "usage: $(basename $0) ..." + msg "usage: ${0##*/} ..." exit 1 fi diff --git a/db-remove b/db-remove index 255aa32..4e45881 100755 --- a/db-remove +++ b/db-remove @@ -1,10 +1,10 @@ #!/bin/bash -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" if [ $# -lt 3 ]; then - msg "usage: $(basename $0) ..." + msg "usage: ${0##*/} ..." exit 1 fi diff --git a/db-repo-add b/db-repo-add index b83fb77..2b7894a 100755 --- a/db-repo-add +++ b/db-repo-add @@ -1,10 +1,10 @@ #!/bin/bash -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" if [ $# -lt 3 ]; then - msg "usage: $(basename $0) ..." + msg "usage: ${0##*/} ..." exit 1 fi diff --git a/db-repo-remove b/db-repo-remove index 4f04ed1..09e34da 100755 --- a/db-repo-remove +++ b/db-repo-remove @@ -1,10 +1,10 @@ #!/bin/bash -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" if [ $# -lt 3 ]; then - msg "usage: $(basename $0) ..." + msg "usage: ${0##*/} ..." exit 1 fi diff --git a/db-sync b/db-sync index e8481d6..c4d495e 100755 --- a/db-sync +++ b/db-sync @@ -21,7 +21,7 @@ ${VERBOSE} && extra="-v" # Returns contents of a repo get_repos() { - mkdir -p ${TMPDIR}/$0.$$.cache + mkdir -p ${TMPDIR}/${0##*/}.$$.cache # Exclude everything but db files rsync ${extra} -mrtlH --no-p --include="*/" \ --include="*.db" \ @@ -30,7 +30,7 @@ get_repos() { --include="*${FILESEXT}" \ --exclude="*" \ --delete-after \ - rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/$0.$$.cache + rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/${0##*/}.$$.cache } get_repo_content() { @@ -48,7 +48,7 @@ get_blacklist() { # repo # arch get_repo_file() { - echo "${TMPDIR}/$0.$$.cache/${1}/os/${2}/${1}" + echo "${TMPDIR}/${0##*/}.$$.cache/${1}/os/${2}/${1}" } # Process the databases and get the libre packages @@ -118,7 +118,7 @@ init() { rsync ${extra} -rtlH \ --delay-updates \ --safe-links \ - ${TMPDIR}/$0.$$.cache/${_repo}/os/${_arch}/ \ + ${TMPDIR}/${0##*/}.$$.cache/${_repo}/os/${_arch}/ \ ${FTP_BASE}/${_repo}/os/${_arch}/ # Cleanup @@ -176,9 +176,9 @@ trap_exit() { } -source $(dirname $0)/config -source $(dirname $0)/local_config -source $(dirname $0)/libremessages +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/local_config" +source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg set -E @@ -189,4 +189,4 @@ trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR init -rm -r ${TMPDIR}/$0.$$.cache +rm -r ${TMPDIR}/${0##*/}.$$.cache diff --git a/db-update b/db-update index cdde63b..86faec3 100755 --- a/db-update +++ b/db-update @@ -1,10 +1,10 @@ #!/bin/bash -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" if [ $# -ge 1 ]; then - warning "Calling $(basename $0) with a specific repository is no longer supported" + warning "Calling ${0##*/} with a specific repository is no longer supported" exit 1 fi diff --git a/get-repos b/get-repos index 92db9a2..a477cf2 100755 --- a/get-repos +++ b/get-repos @@ -9,8 +9,8 @@ trap_exit() { exit 1 } -source $(dirname $0)/config -source $(dirname $0)/libremessages +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg set -E @@ -19,7 +19,7 @@ trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR -TMPDIR="$(mktemp -dt "$(basename $0).XXXX")" +TMPDIR="$(mktemp -dt "${0##*/}.XXXX")" DBLIST=() # Repos diff --git a/migrate-repo b/migrate-repo index 2e44adb..751d5bd 100755 --- a/migrate-repo +++ b/migrate-repo @@ -1,6 +1,6 @@ #!/bin/bash -source $(dirname $0)/config +source "$(dirname "$(readlink -e "$0")")/config" #dryrun="--dry-run" diff --git a/mkrepo b/mkrepo index 5f704cc..10d014b 100755 --- a/mkrepo +++ b/mkrepo @@ -3,8 +3,8 @@ # License: GPLv3+ # Description: A script to quickly create new [repos] -source $(dirname $0)/config -source $(dirname $0)/local_config +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/local_config" # TODO it would be simpler to expand arrays to {element1,element2,etc} for repo in $@; do diff --git a/repo-restore-to-normal b/repo-restore-to-normal index 9463731..3636920 100755 --- a/repo-restore-to-normal +++ b/repo-restore-to-normal @@ -1,8 +1,8 @@ #!/bin/bash # Solves issue165 -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" CLEANUP_DESTDIR=/home/parabolavnx/repo/pool/restore PKGREPOS=(community) diff --git a/testing2x b/testing2x index 54cae11..14a45de 100755 --- a/testing2x +++ b/testing2x @@ -1,10 +1,10 @@ #!/bin/bash -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" if [ $# -lt 1 ]; then - msg "usage: $(basename $0) ..." + msg "usage: ${0##*/} ..." exit 1 fi @@ -54,7 +54,7 @@ for repo in 'core' 'extra'; do repo_unlock ${repo} ${pkgarch} done if [ -n "${pkgs[${repo}]}" ]; then - "$(dirname $0)/db-move" 'testing' "${repo}" ${pkgs[${repo}]} + "$(dirname "$(readlink -e "$0")")/db-move" 'testing' "${repo}" ${pkgs[${repo}]} fi done diff --git a/yf-update b/yf-update index 9c2131e..3bce4aa 100755 --- a/yf-update +++ b/yf-update @@ -1,17 +1,17 @@ #!/bin/bash -source $(dirname $0)/local_config -source $(dirname $0)/config -source $(dirname $0)/libremessages +source "$(dirname "$(readlink -e "$0")")/local_config" +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/libremessages" blacklist_mtime=$(printf "%.0f" $(find ${blacklist} -printf "%T@")) -last_bl_mtime=$(cat $(dirname $0)/yftime) +last_bl_mtime=$(cat "$(dirname "$(readlink -e "$0")")/yftime") if [ $blacklist_mtime -gt $last_bl_mtime ]; then - pushd $(dirname $0)/yf + pushd "$(dirname "$(readlink -e "$0")")/yf" makepkg -f find . -name "*${PKGEXT}" -exec mv {} ${STAGING}/libre \; popd - echo ${blacklist_mtime} > $(dirname $0)/yftime + echo ${blacklist_mtime} > "$(dirname "$(readlink -e "$0")")/yftime" msg2 "built and staged" else msg2 "nothing to do" -- cgit v1.2.3-2-g168b From 8650ad4fdef09b1244ba138531b6a1e2bbd0975b Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 18:00:12 -0500 Subject: Get rid of $ARCH_BASE; these days it is the same as $FTP_BASE --- config | 1 - createrepos | 2 +- cron-jobs/sourceballs | 10 +++++----- cron-jobs/sourceballs2 | 2 +- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/config b/config index 9d72ebd..c3f9a55 100644 --- a/config +++ b/config @@ -1,6 +1,5 @@ #!/bin/bash # as a hint to text editors FTP_BASE="/srv/http/repo/public" -ARCH_BASE="/srv/http/repo/public" SVNREPO="/var/abs" # Repos from Arch diff --git a/createrepos b/createrepos index a8f93e8..8da2455 100755 --- a/createrepos +++ b/createrepos @@ -3,6 +3,6 @@ source "$(dirname "$(readlink -e "$0")")/config" -mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${ARCH_BASE}" "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" +mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" "$(dirname "$(readlink -e "$0")")/create-repo" "${PKGREPOS[@]}" diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 73d8432..4debcee 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -21,10 +21,10 @@ renice +10 -p $$ > /dev/null for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do # Repo does not exist; skip it - if [ ! -f "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue fi - bsdtar -xOf "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ + bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ | awk '/^%NAME%/ { getline b }; /^%BASE%/ { getline b }; /^%VERSION%/ { getline v }; @@ -46,7 +46,7 @@ for repo in ${PKGREPOS[@]}; do done # Create a list of all available source package file names -find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" +find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" # Check for all packages if we need to build a source package for repo in ${PKGREPOS[@]}; do @@ -96,7 +96,7 @@ for repo in ${PKGREPOS[@]}; do pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null makepkg --nocolor --allsource --ignorearch # >/dev/null 2>&1 if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then - mv "${pkgbase}-${pkgver}${SRCEXT}" "${ARCH_BASE}/${SRCPOOL}" + mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" newpkgs[${#newpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" @@ -132,7 +132,7 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" if ! ${SOURCE_CLEANUP_DRYRUN}; then - mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" fi done diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index eb46579..bbe227d 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -17,7 +17,7 @@ script_lock renice +10 -p $$ > /dev/null # Create a list of all available source package file names -find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" +find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" pushd "${SVNREPO}" >/dev/null -- cgit v1.2.3-2-g168b From 62142bcbac5f457b2172c7311741f90fedf0c776 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 15:20:16 -0500 Subject: fix comments, indentation --- cron-jobs/sourceballs | 12 ++++++------ db-functions | 14 +++++++------- db-move | 2 +- db-remove | 4 ++-- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 4debcee..6393472 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -59,11 +59,11 @@ for repo in ${PKGREPOS[@]}; do pkgarch=${pkginfo[2]} pkglicense=(${pkginfo[@]:3}) - # Should this packages be skipped? + # Should this package be skipped? if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then continue fi - # Commenting out, we'll sourceball everything + # Commenting out, we'll sourceball everything # Check if the license or .force file does not enforce creating a source package # if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then # continue @@ -83,10 +83,10 @@ for repo in ${PKGREPOS[@]}; do #svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ # "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 - # If it's on official repos, nor [libre], nor [libre-testing] - cp -r "${SVNREPO}/$repo/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ - cp -r "${SVNREPO}/libre/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ - cp -r "${SVNREPO}/libre-testing/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 + # If it's on official repos, nor [libre], nor [libre-testing] + cp -r "${SVNREPO}/$repo/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ + cp -r "${SVNREPO}/libre/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ + cp -r "${SVNREPO}/libre-testing/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 if [ $? -ge 1 ]; then failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" continue diff --git a/db-functions b/db-functions index 26b23e4..e681504 100644 --- a/db-functions +++ b/db-functions @@ -375,7 +375,7 @@ check_splitpkgs() { for pkgfile in ${pkgfiles[@]}; do issplitpkg "${pkgfile}" || continue local _pkgbase="$(getpkgbase ${pkgfile})" - msg2 "Checking $_pkgbase" + msg2 "Checking $_pkgbase" local _pkgname="$(getpkgname ${pkgfile})" local _pkgarch="$(getpkgarch ${pkgfile})" mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" @@ -384,14 +384,14 @@ check_splitpkgs() { if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" - cp -r ${SVNREPO}/$repo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ - cp -r ${SVNREPO}/libre/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ - cp -r ${SVNREPO}/libre-testing/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/$_pkgbase">/dev/null 2>&1 + cp -r ${SVNREPO}/$repo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ + cp -r ${SVNREPO}/libre/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ + cp -r ${SVNREPO}/libre-testing/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/$_pkgbase">/dev/null 2>&1 [[ $? -ge 1 ]] && { - echo "Failed $_pkgbase-$_pkgver-$_pkgarch" - return 1 - } + echo "Failed $_pkgbase-$_pkgver-$_pkgarch" + return 1 + } fi local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) diff --git a/db-move b/db-move index a3f1532..41ef4c1 100755 --- a/db-move +++ b/db-move @@ -24,7 +24,7 @@ for pkgarch in ${ARCHES[@]}; do repo_lock ${repo_from} ${pkgarch} || exit 1 done -# No idea why we loop twice... -- fauno +# First loop is to check that all necessary files exist for pkgbase in ${args[@]:2}; do for pkgarch in ${ARCHES[@]} 'any'; do svnrepo_from="${SVNREPO}/${repo_from}/${pkgbase}" diff --git a/db-remove b/db-remove index 4e45881..2a12dba 100755 --- a/db-remove +++ b/db-remove @@ -30,8 +30,8 @@ remove_pkgs=() for pkgbase in "${pkgbases[@]}"; do msg "Removing $pkgbase from [$repo]..." - if [ -d "${SVNREPO}/$repo/$pkgbase" ]; then - remove_pkgs=($(. "${SVNREPO}/$repo/$pkgbase/PKGBUILD"; echo ${pkgname[@]})) + if [ -d "${SVNREPO}/$repo/$pkgbase" ]; then + remove_pkgs=($(. "${SVNREPO}/$repo/$pkgbase/PKGBUILD"; echo ${pkgname[@]})) else warning "$pkgbase not found in ABS(libre)" warning "Removing only $pkgbase from the repo" -- cgit v1.2.3-2-g168b From eefb787983d2608511214bcd682f3d8271bac60c Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 16:44:51 -0500 Subject: Normalize to load config then local_config then db-functions --- create-repo | 2 +- cron-jobs/ftpdir-cleanup | 2 +- cron-jobs/integrity-check | 2 +- cron-jobs/repo-sanity-check | 2 +- cron-jobs/sourceballs | 2 +- cron-jobs/sourceballs2 | 2 +- cron-jobs/update-web-db | 2 +- db-check-nonfree | 2 +- db-list-unsigned-packages | 2 +- db-move | 2 +- db-remove | 2 +- db-repo-add | 2 +- db-repo-remove | 2 +- db-update | 2 +- repo-restore-to-normal | 2 +- testing2x | 2 +- yf-update | 2 +- 17 files changed, 17 insertions(+), 17 deletions(-) diff --git a/create-repo b/create-repo index 24b890d..21a2a9c 100755 --- a/create-repo +++ b/create-repo @@ -1,8 +1,8 @@ #!/bin/bash # Creates repository structure -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -eq 0 ]; then msg "Usage: ${0##*/} repo1 [repo2 ... repoX]" diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 8d691b5..b290138 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$(readlink -e "$0")")/../db-functions" . "$(dirname "$(readlink -e "$0")")/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" clean_pkg() { local pkg diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index 05a56a5..86a8f1d 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -2,8 +2,8 @@ dirname="$(dirname "$(readlink -e "$0")")" -. "${dirname}/../db-functions" . "${dirname}/../config" +. "${dirname}/../db-functions" script_lock diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index 2aa7892..ee4c061 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -1,8 +1,8 @@ #!/bin/bash # Solves issue165 -. "$(dirname "$(readlink -e "$0")")/../db-functions" . "$(dirname "$(readlink -e "$0")")/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" # Traverse all repos for _repo in ${PKGREPOS[@]}; do diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 6393472..c9aadba 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -1,8 +1,8 @@ #!/bin/bash dirname="$(dirname "$(readlink -e "$0")")" -. "${dirname}/../db-functions" . "${dirname}/../config" +. "${dirname}/../db-functions" pushd "${WORKDIR}" >/dev/null script_lock diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index bbe227d..1432bdf 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -5,8 +5,8 @@ # Remove the old sourceballs dirname="$(dirname "$(readlink -e "$0")")" -. "${dirname}/../db-functions" . "${dirname}/../config" +. "${dirname}/../db-functions" . "${MAKEPKGCONF}" pushd "${WORKDIR}" >/dev/null diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 825eea6..713e75e 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$(readlink -e "$0")")/../db-functions" . "$(dirname "$(readlink -e "$0")")/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" # setup paths SPATH="/srv/http/archweb" diff --git a/db-check-nonfree b/db-check-nonfree index 661daa6..5cb7f6f 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -ge 1 ]; then warning "Calling ${0##*/} with a specific repository is not supported" diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages index 985d1c0..5105096 100755 --- a/db-list-unsigned-packages +++ b/db-list-unsigned-packages @@ -20,8 +20,8 @@ set -e # unsigned packages available for architecture $1 and specified for # architecture $2 (usually $1 or any, default is to list all). -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 1 ]; then msg "usage: ${0##*/} " diff --git a/db-move b/db-move index 41ef4c1..eed48eb 100755 --- a/db-move +++ b/db-move @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 3 ]; then msg "usage: ${0##*/} ..." diff --git a/db-remove b/db-remove index 2a12dba..46585ad 100755 --- a/db-remove +++ b/db-remove @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 3 ]; then msg "usage: ${0##*/} ..." diff --git a/db-repo-add b/db-repo-add index 2b7894a..a6355a1 100755 --- a/db-repo-add +++ b/db-repo-add @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 3 ]; then msg "usage: ${0##*/} ..." diff --git a/db-repo-remove b/db-repo-remove index 09e34da..7077d62 100755 --- a/db-repo-remove +++ b/db-repo-remove @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 3 ]; then msg "usage: ${0##*/} ..." diff --git a/db-update b/db-update index 86faec3..e3da232 100755 --- a/db-update +++ b/db-update @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -ge 1 ]; then warning "Calling ${0##*/} with a specific repository is no longer supported" diff --git a/repo-restore-to-normal b/repo-restore-to-normal index 3636920..3fe4816 100755 --- a/repo-restore-to-normal +++ b/repo-restore-to-normal @@ -1,8 +1,8 @@ #!/bin/bash # Solves issue165 -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" CLEANUP_DESTDIR=/home/parabolavnx/repo/pool/restore PKGREPOS=(community) diff --git a/testing2x b/testing2x index 14a45de..b6828fc 100755 --- a/testing2x +++ b/testing2x @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$(readlink -e "$0")")/db-functions" . "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if [ $# -lt 1 ]; then msg "usage: ${0##*/} ..." diff --git a/yf-update b/yf-update index 3bce4aa..b6dff14 100755 --- a/yf-update +++ b/yf-update @@ -1,6 +1,6 @@ #!/bin/bash -source "$(dirname "$(readlink -e "$0")")/local_config" source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/local_config" source "$(dirname "$(readlink -e "$0")")/libremessages" blacklist_mtime=$(printf "%.0f" $(find ${blacklist} -printf "%T@")) -- cgit v1.2.3-2-g168b From 9d9116bd23720cf6c5b27aa39e5cc4c71de1fb26 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 21:27:07 -0500 Subject: Fix some array quoting. --- create-repo | 4 ++-- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 22 +++++++++++----------- cron-jobs/ftpdir-cleanup | 10 +++++----- cron-jobs/repo-sanity-check | 4 ++-- cron-jobs/sourceballs | 28 ++++++++++++++-------------- cron-jobs/sourceballs2 | 4 ++-- db-check-nonfree | 22 +++++++++++----------- db-functions | 26 +++++++++++++------------- db-list-unsigned-packages | 2 +- db-move | 28 ++++++++++++++-------------- db-update | 8 ++++---- get-repos | 6 +++--- repo-restore-to-normal | 6 +++--- 13 files changed, 85 insertions(+), 85 deletions(-) diff --git a/create-repo b/create-repo index 21a2a9c..1ec9798 100755 --- a/create-repo +++ b/create-repo @@ -10,12 +10,12 @@ if [ $# -eq 0 ]; then fi msg "Creating repos..." -for _repo in $@; do +for _repo in "$@"; do msg2 "Creating [${_repo}]" mkdir -p "${FTP_BASE}/staging/${_repo}" || \ error "Failed creating staging dir" - for _arch in ${ARCHES[@]}; do + for _arch in "${ARCHES[@]}"; do mkdir -p "${FTP_BASE}/${_repo}/os/${_arch}" || \ error "Failed creating ${_arch} dir" done diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index 3f92169..c8d8618 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -6,18 +6,18 @@ exit() { return; } splitpkg_overrides=('depends' 'optdepends' 'provides' 'conflicts') -variables=('pkgname' 'pkgbase' 'epoch' 'pkgver' 'pkgrel' 'makedepends' 'arch' ${splitpkg_overrides[@]}) +variables=('pkgname' 'pkgbase' 'epoch' 'pkgver' 'pkgrel' 'makedepends' 'arch' "${splitpkg_overrides[@]}") readonly -a variables splitpkg_overrides backup_package_variables() { - for var in ${splitpkg_overrides[@]}; do + for var in "${splitpkg_overrides[@]}"; do indirect="${var}_backup" eval "${indirect}=(\${$var[@]})" done } restore_package_variables() { - for var in ${splitpkg_overrides[@]}; do + for var in "${splitpkg_overrides[@]}"; do indirect="${var}_backup" if [ -n "${!indirect}" ]; then eval "${var}=(\${$indirect[@]})" @@ -42,31 +42,31 @@ print_info() { if [ -n "$arch" ]; then echo "%ARCH%" - for i in ${arch[@]}; do echo $i; done + for i in "${arch[@]}"; do echo $i; done echo "" fi if [ -n "$depends" ]; then echo "%DEPENDS%" - for i in ${depends[@]}; do + for i in "${depends[@]}"; do echo $i done echo "" fi if [ -n "$makedepends" ]; then echo "%MAKEDEPENDS%" - for i in ${makedepends[@]}; do + for i in "${makedepends[@]}"; do echo $i done echo "" fi if [ -n "$conflicts" ]; then echo "%CONFLICTS%" - for i in ${conflicts[@]}; do echo $i; done + for i in "${conflicts[@]}"; do echo $i; done echo "" fi if [ -n "$provides" ]; then echo "%PROVIDES%" - for i in ${provides[@]}; do echo $i; done + for i in "${provides[@]}"; do echo $i; done echo "" fi } @@ -75,7 +75,7 @@ source_pkgbuild() { ret=0 dir=$1 pkgbuild=$dir/PKGBUILD - for var in ${variables[@]}; do + for var in "${variables[@]}"; do unset ${var} done source $pkgbuild &>/dev/null || ret=$? @@ -88,7 +88,7 @@ source_pkgbuild() { if [ "${#pkgname[@]}" -gt "1" ]; then pkgbase=${pkgbase:-${pkgname[0]}} - for pkg in ${pkgname[@]}; do + for pkg in "${pkgname[@]}"; do if [ "$(type -t package_${pkg})" != "function" ]; then echo -e "%INVALID%\n$pkgbuild\n" return 1 @@ -98,7 +98,7 @@ source_pkgbuild() { while IFS= read -r line; do var=${line%%=*} var="${var#"${var%%[![:space:]]*}"}" # remove leading whitespace characters - for realvar in ${variables[@]}; do + for realvar in "${variables[@]}"; do if [ "$var" == "$realvar" ]; then eval $line break diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index b290138..6455ed7 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -25,8 +25,8 @@ clean_pkg() { ${CLEANUP_DRYRUN} && warning 'dry run mode is active' -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue fi @@ -69,7 +69,7 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then fi # cleanup of legacy $repo/os/any directories -for repo in ${PKGREPOS[@]}; do +for repo in "${PKGREPOS[@]}"; do if [ ! -d "${FTP_BASE}/${repo}/os/any" ]; then continue fi @@ -103,8 +103,8 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then done fi -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do repo_unlock ${repo} ${arch} done done diff --git a/cron-jobs/repo-sanity-check b/cron-jobs/repo-sanity-check index ee4c061..9d351df 100755 --- a/cron-jobs/repo-sanity-check +++ b/cron-jobs/repo-sanity-check @@ -5,7 +5,7 @@ . "$(dirname "$(readlink -e "$0")")/../db-functions" # Traverse all repos -for _repo in ${PKGREPOS[@]}; do +for _repo in "${PKGREPOS[@]}"; do msg "Cleaning up [${_repo}]" # Find all pkgnames on this repo's abs @@ -19,7 +19,7 @@ for _repo in ${PKGREPOS[@]}; do >/dev/null 2>&1 # also cleanup package functions - for _pkg in ${pkgname[@]}; do + for _pkg in "${pkgname[@]}"; do unset package_${pkg} >/dev/null 2>&1 done diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index c9aadba..8171980 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -7,8 +7,8 @@ pushd "${WORKDIR}" >/dev/null script_lock -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do repo_lock ${repo} ${arch} || exit 1 done done @@ -18,8 +18,8 @@ renice +10 -p $$ > /dev/null # Create a readable file for each repo with the following format # - [ ] -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do # Repo does not exist; skip it if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue @@ -39,8 +39,8 @@ for repo in ${PKGREPOS[@]}; do done | sort -u > "${WORKDIR}/db-${repo}" done -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do repo_unlock ${repo} ${arch} done done @@ -49,15 +49,15 @@ done find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" # Check for all packages if we need to build a source package -for repo in ${PKGREPOS[@]}; do +for repo in "${PKGREPOS[@]}"; do newpkgs=() failedpkgs=() while read line; do - pkginfo=(${line}) + pkginfo=("${line}") pkgbase=${pkginfo[0]} pkgver=${pkginfo[1]} pkgarch=${pkginfo[2]} - pkglicense=(${pkginfo[@]:3}) + pkglicense=("${pkginfo[@]:3}") # Should this package be skipped? if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then @@ -88,7 +88,7 @@ for repo in ${PKGREPOS[@]}; do cp -r "${SVNREPO}/libre/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ cp -r "${SVNREPO}/libre-testing/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 if [ $? -ge 1 ]; then - failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" + failedpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") continue fi @@ -109,13 +109,13 @@ for repo in ${PKGREPOS[@]}; do if [ ${#newpkgs[@]} -ge 1 ]; then msg "Adding source packages for [${repo}]..." - for new_pkg in ${newpkgs[@]}; do + for new_pkg in "${newpkgs[@]}"; do msg2 "${new_pkg}" done fi if [ ${#failedpkgs[@]} -ge 1 ]; then msg "Failed to create source packages for [${repo}]..." - for failed_pkg in ${failedpkgs[@]}; do + for failed_pkg in "${failedpkgs[@]}"; do msg2 "${failed_pkg}" done fi @@ -129,7 +129,7 @@ old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-s if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old source packages..." ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' - for old_pkg in ${old_pkgs[@]}; do + for old_pkg in "${old_pkgs[@]}"; do msg2 "${old_pkg}" if ! ${SOURCE_CLEANUP_DRYRUN}; then mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" @@ -141,7 +141,7 @@ fi old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -name "*${SRCEXT}" -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old source packages from the cleanup directory..." - for old_pkg in ${old_pkgs[@]}; do + for old_pkg in "${old_pkgs[@]}"; do msg2 "${old_pkg}" ${SOURCE_CLEANUP_DRYRUN} || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" done diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 index 1432bdf..2a26e6a 100755 --- a/cron-jobs/sourceballs2 +++ b/cron-jobs/sourceballs2 @@ -21,7 +21,7 @@ find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort pushd "${SVNREPO}" >/dev/null -for repo in ${PKGREPOS[@]}; do +for repo in "${PKGREPOS[@]}"; do msg "Sourceballing [${repo}]" pushd $repo >/dev/null @@ -40,7 +40,7 @@ for repo in ${PKGREPOS[@]}; do unset build package url pkgdesc source md5sums depends makedepends \ optdepends license arch options check mksource - for _pkg in ${pkgname[@]}; do + for _pkg in "${pkgname[@]}"; do unset package_${_pkg} >/dev/null 2>&1 done diff --git a/db-check-nonfree b/db-check-nonfree index 5cb7f6f..6e2dc17 100755 --- a/db-check-nonfree +++ b/db-check-nonfree @@ -9,16 +9,16 @@ if [ $# -ge 1 ]; then fi # TODO: this might lock too much (architectures) -for repo in ${repos[@]}; do - for pkgarch in ${ARCHES[@]}; do +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do repo_lock ${repo} ${pkgarch} || exit 1 done done msg "Check nonfree in repo:" nonfree=($(cut -d: -f1 ${BLACKLIST_FILE} | sort -u)) -for repo in ${ARCHREPOS[@]}; do - for pkgarch in ${ARCHES[@]}; do +for repo in "${ARCHREPOS[@]}"; do + for pkgarch in "${ARCHES[@]}"; do msg2 "$repo $pkgarch" if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then continue @@ -27,20 +27,20 @@ for repo in ${ARCHREPOS[@]}; do unset cleanpkgs cleanpkgs=() dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) - for pkgname in ${dbpkgs[@]}; do - if in_array ${pkgname} ${nonfree[@]}; then - cleanpkgs+=(${pkgname}) + for pkgname in "${dbpkgs[@]}"; do + if in_array "${pkgname}" "${nonfree[@]}"; then + cleanpkgs+=("${pkgname}") fi done if [ ${#cleanpkgs[@]} -ge 1 ]; then - msg2 "Nonfree: ${cleanpkgs[@]}" - arch_repo_remove "${repo}" "${pkgarch}" ${cleanpkgs[@]} + msg2 "Nonfree: ${cleanpkgs[*]}" + arch_repo_remove "${repo}" "${pkgarch}" "${cleanpkgs[@]}" fi done done -for repo in ${repos[@]}; do - for pkgarch in ${ARCHES[@]}; do +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do repo_unlock ${repo} ${pkgarch} done done diff --git a/db-functions b/db-functions index e681504..83e0613 100644 --- a/db-functions +++ b/db-functions @@ -115,7 +115,7 @@ cleanup() { local arch trap - EXIT INT QUIT TERM - for l in ${LOCKS[@]}; do + for l in "${LOCKS[@]}"; do repo=${l%.*} arch=${l#*.} if [ -d "$TMPDIR/.repolock.$repo.$arch" ]; then @@ -299,7 +299,7 @@ getpkgfiles() { exit 1 fi - for f in ${@}; do + for f in "${@}"; do if [ ! -f "${f}" ]; then error "Package ${f} not found!" exit 1 @@ -309,7 +309,7 @@ getpkgfiles() { fi done - echo ${@} + echo "${@}" } check_pkgfile() { @@ -322,7 +322,7 @@ check_pkgfile() { local pkgarch="$(getpkgarch ${pkgfile})" [ $? -ge 1 ] && return 1 - in_array "${pkgarch}" ${ARCHES[@]} 'any' || return 1 + in_array "${pkgarch}" "${ARCHES[@]}" 'any' || return 1 if echo "$(basename ${pkgfile})" | grep -q "${pkgname}-${pkgver}-${pkgarch}"; then return 0 @@ -343,7 +343,7 @@ check_pkgsvn() { [ $? -ge 1 ] && return 1 local repo="${2}" - in_array "${repo}" ${PKGREPOS[@]} || return 1 + in_array "${repo}" "${PKGREPOS[@]}" || return 1 if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" @@ -356,7 +356,7 @@ check_pkgsvn() { [ "${svnver}" == "${_pkgver}" ] || return 1 local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) - in_array "${_pkgname}" ${svnnames[@]} || return 1 + in_array "${_pkgname}" "${svnnames[@]}" || return 1 return 0 } @@ -364,7 +364,7 @@ check_pkgsvn() { check_splitpkgs() { local repo="${1}" shift - local pkgfiles=(${@}) + local pkgfiles=("${@}") local pkgfile local pkgdir local svnname @@ -372,7 +372,7 @@ check_splitpkgs() { mkdir -p "${WORKDIR}/check_splitpkgs/" pushd "${WORKDIR}/check_splitpkgs" >/dev/null - for pkgfile in ${pkgfiles[@]}; do + for pkgfile in "${pkgfiles[@]}"; do issplitpkg "${pkgfile}" || continue local _pkgbase="$(getpkgbase ${pkgfile})" msg2 "Checking $_pkgbase" @@ -395,7 +395,7 @@ check_splitpkgs() { fi local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) - for svnname in ${svnnames[@]}; do + for svnname in "${svnnames[@]}"; do echo "${svnname}" >> "${repo}/${_pkgarch}/${_pkgbase}/svn" done done @@ -430,8 +430,8 @@ check_pkgrepos() { local repo local arch - for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do + for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} ] && return 1 [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}.sig ] && return 1 [ -f "${FTP_BASE}/${repo}/os/${arch}/$(basename ${pkgfile})" ] && return 1 @@ -493,7 +493,7 @@ set_repo_permission() { arch_repo_add() { local repo=$1 local arch=$2 - local pkgs=(${@:3}) + local pkgs=("${@:3}") # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null @@ -508,7 +508,7 @@ arch_repo_add() { arch_repo_remove() { local repo=$1 local arch=$2 - local pkgs=(${@:3}) + local pkgs=("${@:3}") local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" diff --git a/db-list-unsigned-packages b/db-list-unsigned-packages index 5105096..f593686 100755 --- a/db-list-unsigned-packages +++ b/db-list-unsigned-packages @@ -31,7 +31,7 @@ fi arch=$1 shift -for repo in ${PKGREPOS[@]} +for repo in "${PKGREPOS[@]}" do db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" [ -f "$db" ] && "$(dirname "$(readlink -e "$0")")/db-list-unsigned-packages.py" "$repo" "$@" < "$db" diff --git a/db-move b/db-move index eed48eb..a6abb8f 100755 --- a/db-move +++ b/db-move @@ -8,7 +8,7 @@ if [ $# -lt 3 ]; then exit 1 fi -args=(${@}) +args=("${@}") repo_from="${args[0]}" repo_to="${args[1]}" ftppath_from="${FTP_BASE}/${repo_from}/os/" @@ -19,14 +19,14 @@ if ! check_repo_permission $repo_to || ! check_repo_permission $repo_from; then fi # TODO: this might lock too much (architectures) -for pkgarch in ${ARCHES[@]}; do +for pkgarch in "${ARCHES[@]}"; do repo_lock ${repo_to} ${pkgarch} || exit 1 repo_lock ${repo_from} ${pkgarch} || exit 1 done # First loop is to check that all necessary files exist -for pkgbase in ${args[@]:2}; do - for pkgarch in ${ARCHES[@]} 'any'; do +for pkgbase in "${args[@]:2}"; do + for pkgarch in "${ARCHES[@]}" 'any'; do svnrepo_from="${SVNREPO}/${repo_from}/${pkgbase}" if [ -r "${svnrepo_from}/PKGBUILD" ]; then pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo ${pkgname[@]})) @@ -40,13 +40,13 @@ for pkgbase in ${args[@]:2}; do fi if [ "${pkgarch}" == 'any' ]; then - tarches=(${ARCHES[@]}) + tarches=("${ARCHES[@]}") else tarches=("${pkgarch}") fi - for pkgname in ${pkgnames[@]}; do - for tarch in ${tarches[@]}; do + for pkgname in "${pkgnames[@]}"; do + for tarch in "${tarches[@]}"; do getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT} >/dev/null done done @@ -60,8 +60,8 @@ msg "Moving packages from [${repo_from}] to [${repo_to}]..." declare -A add_pkgs declare -A remove_pkgs -for pkgbase in ${args[@]:2}; do - for pkgarch in ${ARCHES[@]} 'any'; do +for pkgbase in "${args[@]:2}"; do + for pkgarch in "${ARCHES[@]}" 'any'; do svnrepo_from="${SVNREPO}/${repo_from}/${pkgbase}" if [ -f "${svnrepo_from}/PKGBUILD" ]; then @@ -70,12 +70,12 @@ for pkgbase in ${args[@]:2}; do else tarches=("${pkgarch}") fi - msg2 "${pkgbase} ($(echo ${tarches[@]}))" + msg2 "${pkgbase} (${tarches[*]})" pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo ${pkgname[@]})) pkgver=$(. "${svnrepo_from}/PKGBUILD"; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) - for pkgname in ${pkgnames[@]}; do - for tarch in ${tarches[@]}; do + for pkgname in "${pkgnames[@]}"; do + for tarch in "${tarches[@]}"; do pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT}) pkgfile=$(basename "${pkgpath}") @@ -96,14 +96,14 @@ for pkgbase in ${args[@]:2}; do done done -for tarch in ${ARCHES[@]}; do +for tarch in "${ARCHES[@]}"; do if [ -n "${add_pkgs[${tarch}]}" ]; then arch_repo_add "${repo_to}" "${tarch}" ${add_pkgs[${tarch}]} arch_repo_remove "${repo_from}" "${tarch}" ${remove_pkgs[${tarch}]} fi done -for pkgarch in ${ARCHES[@]}; do +for pkgarch in "${ARCHES[@]}"; do repo_unlock ${repo_from} ${pkgarch} repo_unlock ${repo_to} ${pkgarch} done diff --git a/db-update b/db-update index e3da232..60cc6cd 100755 --- a/db-update +++ b/db-update @@ -28,7 +28,7 @@ for repo in "${repos[@]}"; do if [ ${#pkgs[@]} -gt 0 ] && ! check_repo_permission "${repo}"; then die "You don't have permission to update packages in ${repo}" fi - for pkg in ${pkgs[@]}; do + for pkg in "${pkgs[@]}"; do if [ -h "${pkg}" ]; then die "Package ${repo}/$(basename ${pkg}) is a symbolic link" fi @@ -40,7 +40,7 @@ for repo in "${repos[@]}"; do fi done # This is fucking obnoxious - #if ! check_splitpkgs ${repo} ${pkgs[@]}; then + #if ! check_splitpkgs ${repo} "${pkgs[@]}"; then # die "Missing split packages for ${repo}" #fi else @@ -51,10 +51,10 @@ done for repo in "${repos[@]}"; do msg "Updating [${repo}]..." any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXT} 2>/dev/null)) - for pkgarch in ${ARCHES[@]}; do + for pkgarch in "${ARCHES[@]}"; do add_pkgs=() arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-${pkgarch}${PKGEXT} 2>/dev/null)) - for pkg in ${arch_pkgs[@]} ${any_pkgs[@]}; do + for pkg in "${arch_pkgs[@]}" "${any_pkgs[@]}"; do pkgfile="$(basename ${pkg})" msg2 "${pkgfile} (${pkgarch})" # any packages might have been moved by the previous run diff --git a/get-repos b/get-repos index a477cf2..b98d601 100755 --- a/get-repos +++ b/get-repos @@ -23,8 +23,8 @@ TMPDIR="$(mktemp -dt "${0##*/}.XXXX")" DBLIST=() # Repos -for _repo in ${PKGREPOS[@]}; do - for _arch in ${ARCHES[@]}; do +for _repo in "${PKGREPOS[@]}"; do + for _arch in "${ARCHES[@]}"; do DBLIST+=("http://repo.parabolagnulinux.org/${_repo}/os/${_arch}/${_repo}${FILESEXT}") done done @@ -35,7 +35,7 @@ wget --directory-prefix=${TMPDIR} \ --no-verbose \ --force-directories \ --no-host-directories \ - ${DBLIST[@]} || true + "${DBLIST[@]}" || true # Always return true, some databases are expect to be missing # Create the arches regexp arch1|arch2 diff --git a/repo-restore-to-normal b/repo-restore-to-normal index 3fe4816..063aacf 100755 --- a/repo-restore-to-normal +++ b/repo-restore-to-normal @@ -12,7 +12,7 @@ PKGREPOS=(community) # sed "s/^\(.\+-[^-]\+-[^-]\+\)-[^-]\+$/\1/")) # Traverse all repos -for _repo in ${PKGREPOS[@]}; do +for _repo in "${PKGREPOS[@]}"; do msg "Restoring [${_repo}]" # Find all pkgnames on this repo's abs @@ -27,7 +27,7 @@ for _repo in ${PKGREPOS[@]}; do >/dev/null 2>&1 # also cleanup package functions - for _pkg in ${pkgname[@]}; do + for _pkg in "${pkgname[@]}"; do unset package_${pkg} >/dev/null 2>&1 # this fills the on_abs array echo ${_pkg}-${pkgver}-${pkgrel} @@ -49,7 +49,7 @@ for _repo in ${PKGREPOS[@]}; do msg2 "Restoring the following packages:" # plain "$(echo ${restore[@]} | tr ' ' "\n")" - for _pkg in ${on_abs[@]}; do + for _pkg in "${on_abs[@]}"; do find ${CLEANUP_DESTDIR} -name "${_pkg}*" -exec cp -v '{}' ${STAGING}/${_repo} \; done -- cgit v1.2.3-2-g168b From aefd5a4d961e3eca3326c90ba266d04b325c1f15 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 20:36:20 -0500 Subject: db-sync: use tab indent --- db-sync | 263 ++++++++++++++++++++++++++++++++-------------------------------- 1 file changed, 131 insertions(+), 132 deletions(-) diff --git a/db-sync b/db-sync index c4d495e..0e07757 100755 --- a/db-sync +++ b/db-sync @@ -21,158 +21,157 @@ ${VERBOSE} && extra="-v" # Returns contents of a repo get_repos() { - mkdir -p ${TMPDIR}/${0##*/}.$$.cache -# Exclude everything but db files - rsync ${extra} -mrtlH --no-p --include="*/" \ - --include="*.db" \ - --include="*${DBEXT}" \ - --include="*.files" \ - --include="*${FILESEXT}" \ - --exclude="*" \ - --delete-after \ - rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/${0##*/}.$$.cache + mkdir -p ${TMPDIR}/${0##*/}.$$.cache + # Exclude everything but db files + rsync ${extra} -mrtlH --no-p --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --include="*.files" \ + --include="*${FILESEXT}" \ + --exclude="*" \ + --delete-after \ + rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/${0##*/}.$$.cache } get_repo_content() { -# Return all contents - bsdtar tf ${1} | \ - cut -d "/" -f 1 | \ - sort -u + # Return all contents + bsdtar tf ${1} | \ + cut -d "/" -f 1 | \ + sort -u } # Prints blacklisted packages get_blacklist() { - cut -d ':' -f 1 "${BLACKLIST_FILE}" + cut -d ':' -f 1 "${BLACKLIST_FILE}" } # repo # arch get_repo_file() { - echo "${TMPDIR}/${0##*/}.$$.cache/${1}/os/${2}/${1}" + echo "${TMPDIR}/${0##*/}.$$.cache/${1}/os/${2}/${1}" } # Process the databases and get the libre packages init() { -# Get the blacklisted packages - blacklist=($(get_blacklist)) -# Store all the whitelist files - whitelists=() - - msg "${#blacklist[@]} packages in blacklist" - -# Sync the repos databases - get_repos - -# Traverse all repo-arch pairs - for _repo in ${ARCHREPOS[@]}; do - for _arch in ${ARCHARCHES[@]}; do - msg "Processing ${_repo}-${_arch}" - - db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT} - files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT} - - if [ ! -f "${db_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${db_file}" - continue - fi - if [ ! -f "${files_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${files_file}" - continue - fi - -# Remove blacklisted packages and count them -# TODO capture all removed packages for printing on debug mode - msg2 "Removing blacklisted packages from .db database..." - LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" - msg2 "Removing blacklisted packages from .files database..." - LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" - -# Get db contents - db=($(get_repo_content ${db_file})) - - msg2 "Process clean db for syncing..." - -# Create a whitelist, add * wildcard to end -# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if -# we aren't syncing them - echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist - - msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" - -# Sync excluding everything but whitelist -# We delete here for cleanup - rsync ${extra} -rtlH \ - --delete-after \ - --delete-excluded \ - --delay-updates \ - --include-from=/tmp/${_repo}-${_arch}.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ - -# Add a new whitelist - whitelists+=(/tmp/${_repo}-${_arch}.whitelist) - - msg "Putting databases back in place" - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - ${TMPDIR}/${0##*/}.$$.cache/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ - -# Cleanup - unset db - done - done - - - msg "Syncing package pool" -# Concatenate all whitelists - cat ${whitelists[@]} | sort -u > /tmp/any.whitelist - - msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" - -# Sync -# *Don't delete-after*, this is the job of cleanup scripts. It will remove our -# packages too - for PKGPOOL in ${PKGPOOLS[@]}; do - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ - ${FTP_BASE}/${PKGPOOL}/ - done - -# Sync sources - msg "Syncing source pool" - #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist - - #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool" -# Sync -# *Don't delete-after*, this is the job of cleanup scripts. It will remove our -# packages too - for SRCPOOL in ${SRCPOOLS[@]}; do - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ - ${FTP_BASE}/${SRCPOOL}/ - done - - -# Cleanup - unset blacklist whitelists _arch _repo repo_file + # Get the blacklisted packages + blacklist=($(get_blacklist)) + # Store all the whitelist files + whitelists=() + + msg "${#blacklist[@]} packages in blacklist" + + # Sync the repos databases + get_repos + + # Traverse all repo-arch pairs + for _repo in ${ARCHREPOS[@]}; do + for _arch in ${ARCHARCHES[@]}; do + msg "Processing ${_repo}-${_arch}" + + db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT} + files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT} + + if [ ! -f "${db_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${db_file}" + continue + fi + if [ ! -f "${files_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${files_file}" + continue + fi + + # Remove blacklisted packages and count them + # TODO capture all removed packages for printing on debug mode + msg2 "Removing blacklisted packages from .db database..." + LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" + msg2 "Removing blacklisted packages from .files database..." + LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" + + # Get db contents + db=($(get_repo_content ${db_file})) + + msg2 "Process clean db for syncing..." + + # Create a whitelist, add * wildcard to end + # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if + # we aren't syncing them + echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist + + msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" + + # Sync excluding everything but whitelist + # We delete here for cleanup + rsync ${extra} -rtlH \ + --delete-after \ + --delete-excluded \ + --delay-updates \ + --include-from=/tmp/${_repo}-${_arch}.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + + # Add a new whitelist + whitelists+=(/tmp/${_repo}-${_arch}.whitelist) + + msg "Putting databases back in place" + rsync ${extra} -rtlH \ + --delay-updates \ + --safe-links \ + ${TMPDIR}/${0##*/}.$$.cache/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + + # Cleanup + unset db + done + done + + + msg "Syncing package pool" + # Concatenate all whitelists + cat ${whitelists[@]} | sort -u > /tmp/any.whitelist + + msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" + + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + for PKGPOOL in ${PKGPOOLS[@]}; do + rsync ${extra} -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ + ${FTP_BASE}/${PKGPOOL}/ + done + + # Sync sources + msg "Syncing source pool" + #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist + + #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool" + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + for SRCPOOL in ${SRCPOOLS[@]}; do + rsync ${extra} -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ + ${FTP_BASE}/${SRCPOOL}/ + done + + # Cleanup + unset blacklist whitelists _arch _repo repo_file } trap_exit() { - echo - error "$@" - exit 1 + echo + error "$@" + exit 1 } -- cgit v1.2.3-2-g168b From d3afe9bb766bbdfaab1474d21fd5e28f0a356039 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 20:49:48 -0500 Subject: Avoid using $(basename $var) , use ${var##*/} instead --- abslibre | 2 +- cron-jobs/ftpdir-cleanup | 2 +- db-functions | 10 +++++----- db-move | 2 +- db-update | 8 ++++---- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/abslibre b/abslibre index fa777ac..a5733f0 100755 --- a/abslibre +++ b/abslibre @@ -96,7 +96,7 @@ sync_pre_mips64el() { # Create .abs.tar.gz tarballs create_tarballs() { for repo in ${ABSLIBRE}/{i686,x86_64}/*; do - baserepo=$(basename $repo) + baserepo=${repo##*/} arch=$(basename $(dirname $repo)) # Remove the old one diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 6455ed7..cb14382 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -16,7 +16,7 @@ clean_pkg() { if [ -e "$pkg.sig" ]; then mv -f "$pkg.sig" "$CLEANUP_DESTDIR" fi - touch "${CLEANUP_DESTDIR}/$(basename ${pkg})" + touch "${CLEANUP_DESTDIR}/${pkg##*/}" fi done fi diff --git a/db-functions b/db-functions index 83e0613..52ad8bb 100644 --- a/db-functions +++ b/db-functions @@ -324,7 +324,7 @@ check_pkgfile() { in_array "${pkgarch}" "${ARCHES[@]}" 'any' || return 1 - if echo "$(basename ${pkgfile})" | grep -q "${pkgname}-${pkgver}-${pkgarch}"; then + if echo "${pkgfile##*/}" | grep -q "${pkgname}-${pkgver}-${pkgarch}"; then return 0 else return 1 @@ -425,8 +425,8 @@ check_pkgrepos() { [ -f "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} ] && return 1 [ -f "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}.sig ] && return 1 - [ -f "${FTP_BASE}/${PKGPOOL}/$(basename ${pkgfile})" ] && return 1 - [ -f "${FTP_BASE}/${PKGPOOL}/$(basename ${pkgfile}).sig" ] && return 1 + [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile##*/}" ] && return 1 + [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile##*/}.sig" ] && return 1 local repo local arch @@ -434,8 +434,8 @@ check_pkgrepos() { for arch in "${ARCHES[@]}"; do [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} ] && return 1 [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}.sig ] && return 1 - [ -f "${FTP_BASE}/${repo}/os/${arch}/$(basename ${pkgfile})" ] && return 1 - [ -f "${FTP_BASE}/${repo}/os/${arch}/$(basename ${pkgfile}).sig" ] && return 1 + [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgfile##*/}" ] && return 1 + [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgfile##*/}.sig" ] && return 1 done done diff --git a/db-move b/db-move index a6abb8f..f1c3dea 100755 --- a/db-move +++ b/db-move @@ -77,7 +77,7 @@ for pkgbase in "${args[@]:2}"; do for pkgname in "${pkgnames[@]}"; do for tarch in "${tarches[@]}"; do pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT}) - pkgfile=$(basename "${pkgpath}") + pkgfile="${pkgpath##*/}" # copy package to pool if needed # TODO: can be removed once every package has been moved to the package pool diff --git a/db-update b/db-update index 60cc6cd..99c8be5 100755 --- a/db-update +++ b/db-update @@ -30,13 +30,13 @@ for repo in "${repos[@]}"; do fi for pkg in "${pkgs[@]}"; do if [ -h "${pkg}" ]; then - die "Package ${repo}/$(basename ${pkg}) is a symbolic link" + die "Package ${repo}/${pkg##*/} is a symbolic link" fi if ! check_pkgfile "${pkg}"; then - die "Package ${repo}/$(basename ${pkg}) is not consistent with its meta data" + die "Package ${repo}/${pkg##*/} is not consistent with its meta data" fi if ! check_pkgrepos "${pkg}"; then - die "Package ${repo}/$(basename ${pkg}) already exists in another repository" + die "Package ${repo}/${pkg##*/} already exists in another repository" fi done # This is fucking obnoxious @@ -55,7 +55,7 @@ for repo in "${repos[@]}"; do add_pkgs=() arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-${pkgarch}${PKGEXT} 2>/dev/null)) for pkg in "${arch_pkgs[@]}" "${any_pkgs[@]}"; do - pkgfile="$(basename ${pkg})" + pkgfile="${pkg##*/}" msg2 "${pkgfile} (${pkgarch})" # any packages might have been moved by the previous run if [ -f "${pkg}" ]; then -- cgit v1.2.3-2-g168b From a1fe8b8d0b49b9285189b1e8e2b5858b0fa4e2aa Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 8 Jan 2014 21:13:03 -0500 Subject: remove unused variables from local_config --- local_config | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/local_config b/local_config index 8f3946a..9b4415e 100644 --- a/local_config +++ b/local_config @@ -1,4 +1,7 @@ -# Mirror options +#/bin/bash # as a hint to text editors +_paraboladir=/srv/http/repo/public + +# db-sync #mirror="mirrors.uk2.net" mirror="mirrors.kernel.org" #mirror="mirror.umd.edu" @@ -7,25 +10,11 @@ mirror="mirrors.kernel.org" #mirror="mirror.de.leaseweb.net" mirrorpath="archlinux" -# Directories: they should end without / -paraboladir=/srv/http/repo/public -tempdir=/tmp -archdb=${tempdir}/db -docs_dir=${paraboladir}/docs -repodir=${paraboladir} -licenses_dir=${docs_dir}/pending_licenses -# End Directories - -# Files -logname=${paraboladir}/log/$(date -u +%Y%m%d-%H:%M)-repo-maintainer.log -rsout_file=${tempdir}/rsout -rsync_not_needed=${tempdir}/rsync_not_needed - -rsync_blacklist=${docs_dir}/rsyncBlacklist +# mkrepo +repodir=${_paraboladir} -blacklist=${docs_dir}/blacklist.txt -whitelist=${docs_dir}/whitelist.txt +# yf-update +blacklist=${_paraboladir}/docs/blacklist.txt +whitelist=${_paraboladir}/docs/whitelist.txt -# Rsync commands -rsync_list_command="rsync -rptgoL --exclude='*.abs.tar.*' --list-only --no-motd " -rsync_update_command="rsync -vrptgoL --exclude='*.abs.tar.*' --no-motd " +unset _paraboladir -- cgit v1.2.3-2-g168b From 93eb77cbad072a25b9aa5542ce7fa59746ca9284 Mon Sep 17 00:00:00 2001 From: aurelien Date: Sat, 11 Jan 2014 12:39:37 +0100 Subject: modification on db-update for mail --- db-update | 1 + 1 file changed, 1 insertion(+) diff --git a/db-update b/db-update index 99c8be5..73d5d3f 100755 --- a/db-update +++ b/db-update @@ -91,5 +91,6 @@ while read -r file; do else mkdir -p -- "${pub%/*}" mv -vn "$file" "$pub" + echo "At `date`, script \"`basename $0`\" mailed to "maintenance@lists.parabolagnulinux.org"." fi done < <(find other sources -type f) -- cgit v1.2.3-2-g168b From 2d217815fcfe05152272c20e49e4e69927d04a35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Wed, 15 Jan 2014 19:12:38 -0200 Subject: revert to old db-sync until fix it, rename lastest db-sync to db-sync.orig --- db-sync | 265 +++++++++++++++++++++++++++++------------------------------ db-sync.orig | 191 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 321 insertions(+), 135 deletions(-) create mode 100755 db-sync.orig diff --git a/db-sync b/db-sync index 0e07757..81dee24 100755 --- a/db-sync +++ b/db-sync @@ -21,163 +21,158 @@ ${VERBOSE} && extra="-v" # Returns contents of a repo get_repos() { - mkdir -p ${TMPDIR}/${0##*/}.$$.cache - # Exclude everything but db files - rsync ${extra} -mrtlH --no-p --include="*/" \ - --include="*.db" \ - --include="*${DBEXT}" \ - --include="*.files" \ - --include="*${FILESEXT}" \ - --exclude="*" \ - --delete-after \ - rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/${0##*/}.$$.cache + mkdir -p ${TMPDIR}/$0.$$.cache +# Exclude everything but db files + rsync ${extra} -mrtlH --no-p --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --exclude="*" \ + --delete-after \ + rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/$0.$$.cache } get_repo_content() { - # Return all contents - bsdtar tf ${1} | \ - cut -d "/" -f 1 | \ - sort -u +# Return all contents + bsdtar tf ${1} | \ + cut -d "/" -f 1 | \ + sort -u } # Prints blacklisted packages get_blacklist() { - cut -d ':' -f 1 "${BLACKLIST_FILE}" + cut -d ':' -f 1 "${BLACKLIST_FILE}" } # repo # arch get_repo_file() { - echo "${TMPDIR}/${0##*/}.$$.cache/${1}/os/${2}/${1}" +# shopt -s nullglob + + echo "${TMPDIR}/$0.$$.cache/${1}/os/${2}/${1}${DBEXT}" } # Process the databases and get the libre packages init() { - # Get the blacklisted packages - blacklist=($(get_blacklist)) - # Store all the whitelist files - whitelists=() - - msg "${#blacklist[@]} packages in blacklist" - - # Sync the repos databases - get_repos - - # Traverse all repo-arch pairs - for _repo in ${ARCHREPOS[@]}; do - for _arch in ${ARCHARCHES[@]}; do - msg "Processing ${_repo}-${_arch}" - - db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT} - files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT} - - if [ ! -f "${db_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${db_file}" - continue - fi - if [ ! -f "${files_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${files_file}" - continue - fi - - # Remove blacklisted packages and count them - # TODO capture all removed packages for printing on debug mode - msg2 "Removing blacklisted packages from .db database..." - LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" - msg2 "Removing blacklisted packages from .files database..." - LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" - - # Get db contents - db=($(get_repo_content ${db_file})) - - msg2 "Process clean db for syncing..." - - # Create a whitelist, add * wildcard to end - # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if - # we aren't syncing them - echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist - - msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" - - # Sync excluding everything but whitelist - # We delete here for cleanup - rsync ${extra} -rtlH \ - --delete-after \ - --delete-excluded \ - --delay-updates \ - --include-from=/tmp/${_repo}-${_arch}.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ - - # Add a new whitelist - whitelists+=(/tmp/${_repo}-${_arch}.whitelist) - - msg "Putting databases back in place" - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - ${TMPDIR}/${0##*/}.$$.cache/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ - - # Cleanup - unset db - done - done - - - msg "Syncing package pool" - # Concatenate all whitelists - cat ${whitelists[@]} | sort -u > /tmp/any.whitelist - - msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" - - # Sync - # *Don't delete-after*, this is the job of cleanup scripts. It will remove our - # packages too - for PKGPOOL in ${PKGPOOLS[@]}; do - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ - ${FTP_BASE}/${PKGPOOL}/ - done - - # Sync sources - msg "Syncing source pool" - #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist - - #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool" - # Sync - # *Don't delete-after*, this is the job of cleanup scripts. It will remove our - # packages too - for SRCPOOL in ${SRCPOOLS[@]}; do - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ - ${FTP_BASE}/${SRCPOOL}/ - done - - # Cleanup - unset blacklist whitelists _arch _repo repo_file +# Get the blacklisted packages + blacklist=($(get_blacklist)) +# Store all the whitelist files + whitelists=() + + msg "${#blacklist[@]} packages in blacklist" + +# Sync the repos databases + get_repos + +# Traverse all repo-arch pairs + for _repo in ${ARCHREPOS[@]}; do + for _arch in ${ARCHARCHES[@]}; do + msg "Processing ${_repo}-${_arch}" + + repo_file=$(get_repo_file ${_repo} ${_arch}) + + if [ ! -f "${repo_file}" ]; then + warning "${repo_file} doesn't exist, skipping this repo-arch" + continue + fi + +# Remove blacklisted packages and count them +# TODO capture all removed packages for printing on debug mode + msg2 "Removing blacklisted packages: $( + LC_ALL=C repo-remove ${repo_file} ${blacklist[@]} 2>&1 | \ + grep "\-> Removing" 2>/dev/null| wc -l)" + +# Get db contents + db=($(get_repo_content ${repo_file})) + + msg2 "Process clean db for syncing..." + +# Create a whitelist, add * wildcard to end +# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if +# we aren't syncing them + echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist + + msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" + +# Sync excluding everything but whitelist +# We delete here for cleanup + rsync ${extra} -rtlH \ + --delete-after \ + --delete-excluded \ + --delay-updates \ + --include-from=/tmp/${_repo}-${_arch}.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + +# Add a new whitelist + whitelists+=(/tmp/${_repo}-${_arch}.whitelist) + + msg "Putting databases back in place" + rsync ${extra} -rtlH \ + --delay-updates \ + --safe-links \ + ${TMPDIR}/$0.$$.cache/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + +# Cleanup + unset db + done + done + + + msg "Syncing package pool" +# Concatenate all whitelists + cat ${whitelists[@]} | sort -u > /tmp/any.whitelist + + msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" + +# Sync +# *Don't delete-after*, this is the job of cleanup scripts. It will remove our +# packages too + for PKGPOOL in ${PKGPOOLS[@]}; do + rsync ${extra} -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ + ${FTP_BASE}/${PKGPOOL}/ + done + +# Sync sources + msg "Syncing source pool" + #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist + + #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool" +# Sync +# *Don't delete-after*, this is the job of cleanup scripts. It will remove our +# packages too + for SRCPOOL in ${SRCPOOLS[@]}; do + rsync ${extra} -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ + ${FTP_BASE}/${SRCPOOL}/ + done + + +# Cleanup + unset blacklist whitelists _arch _repo repo_file } trap_exit() { - echo - error "$@" - exit 1 + echo + error "$@" + exit 1 } -source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/local_config" -source "$(dirname "$(readlink -e "$0")")/libremessages" +source $(dirname $0)/config +source $(dirname $0)/local_config +source $(dirname $0)/libremessages # From makepkg set -E @@ -188,4 +183,4 @@ trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR init -rm -r ${TMPDIR}/${0##*/}.$$.cache +rm -r ${TMPDIR}/$0.$$.cache diff --git a/db-sync.orig b/db-sync.orig new file mode 100755 index 0000000..0e07757 --- /dev/null +++ b/db-sync.orig @@ -0,0 +1,191 @@ +#!/bin/bash +# Syncs Arch repos based on info contained in repo.db files +# License: GPLv3 + +# Principles +# * Get repo.db from an Arch-like repo +# * Generate a list of available packages +# * Create sync whitelist (based on package blacklist) +# * Get packages +# * Check package signatures +# * Check database signatures +# * Sync repo => repo + +# TODO +# * make a tarball of files used for forensics +# * get files db + +# Run as `V=true db-sync` to get verbose output +VERBOSE=${V} +${VERBOSE} && extra="-v" + +# Returns contents of a repo +get_repos() { + mkdir -p ${TMPDIR}/${0##*/}.$$.cache + # Exclude everything but db files + rsync ${extra} -mrtlH --no-p --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --include="*.files" \ + --include="*${FILESEXT}" \ + --exclude="*" \ + --delete-after \ + rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/${0##*/}.$$.cache +} + +get_repo_content() { + # Return all contents + bsdtar tf ${1} | \ + cut -d "/" -f 1 | \ + sort -u +} + +# Prints blacklisted packages +get_blacklist() { + cut -d ':' -f 1 "${BLACKLIST_FILE}" +} + +# repo +# arch +get_repo_file() { + echo "${TMPDIR}/${0##*/}.$$.cache/${1}/os/${2}/${1}" +} + +# Process the databases and get the libre packages +init() { + + # Get the blacklisted packages + blacklist=($(get_blacklist)) + # Store all the whitelist files + whitelists=() + + msg "${#blacklist[@]} packages in blacklist" + + # Sync the repos databases + get_repos + + # Traverse all repo-arch pairs + for _repo in ${ARCHREPOS[@]}; do + for _arch in ${ARCHARCHES[@]}; do + msg "Processing ${_repo}-${_arch}" + + db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT} + files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT} + + if [ ! -f "${db_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${db_file}" + continue + fi + if [ ! -f "${files_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${files_file}" + continue + fi + + # Remove blacklisted packages and count them + # TODO capture all removed packages for printing on debug mode + msg2 "Removing blacklisted packages from .db database..." + LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" + msg2 "Removing blacklisted packages from .files database..." + LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" + + # Get db contents + db=($(get_repo_content ${db_file})) + + msg2 "Process clean db for syncing..." + + # Create a whitelist, add * wildcard to end + # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if + # we aren't syncing them + echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist + + msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" + + # Sync excluding everything but whitelist + # We delete here for cleanup + rsync ${extra} -rtlH \ + --delete-after \ + --delete-excluded \ + --delay-updates \ + --include-from=/tmp/${_repo}-${_arch}.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + + # Add a new whitelist + whitelists+=(/tmp/${_repo}-${_arch}.whitelist) + + msg "Putting databases back in place" + rsync ${extra} -rtlH \ + --delay-updates \ + --safe-links \ + ${TMPDIR}/${0##*/}.$$.cache/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + + # Cleanup + unset db + done + done + + + msg "Syncing package pool" + # Concatenate all whitelists + cat ${whitelists[@]} | sort -u > /tmp/any.whitelist + + msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" + + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + for PKGPOOL in ${PKGPOOLS[@]}; do + rsync ${extra} -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ + ${FTP_BASE}/${PKGPOOL}/ + done + + # Sync sources + msg "Syncing source pool" + #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist + + #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool" + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + for SRCPOOL in ${SRCPOOLS[@]}; do + rsync ${extra} -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ + ${FTP_BASE}/${SRCPOOL}/ + done + + # Cleanup + unset blacklist whitelists _arch _repo repo_file +} + +trap_exit() { + echo + error "$@" + exit 1 +} + + +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/local_config" +source "$(dirname "$(readlink -e "$0")")/libremessages" + +# From makepkg +set -E + +trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT +trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT +trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR + +init + +rm -r ${TMPDIR}/${0##*/}.$$.cache -- cgit v1.2.3-2-g168b From 0684ad038134d7c9ccbaebc709c4267a904b98b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Wed, 15 Jan 2014 19:31:03 -0200 Subject: db-update: add shopt -s nullglob and remove mail script to maintenance@lists.parabolagnulinux.org because it's unstable --- db-update | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db-update b/db-update index 73d5d3f..186ed55 100755 --- a/db-update +++ b/db-update @@ -3,6 +3,8 @@ . "$(dirname "$(readlink -e "$0")")/config" . "$(dirname "$(readlink -e "$0")")/db-functions" +shopt -s nullglob + if [ $# -ge 1 ]; then warning "Calling ${0##*/} with a specific repository is no longer supported" exit 1 @@ -91,6 +93,5 @@ while read -r file; do else mkdir -p -- "${pub%/*}" mv -vn "$file" "$pub" - echo "At `date`, script \"`basename $0`\" mailed to "maintenance@lists.parabolagnulinux.org"." fi done < <(find other sources -type f) -- cgit v1.2.3-2-g168b From 0f9c53d616116cac705b01bfabb2186506aac52a Mon Sep 17 00:00:00 2001 From: Parabola Date: Thu, 16 Jan 2014 04:02:30 +0000 Subject: fix new db-sync --- db-sync | 281 ++++++++++++++++++++++++++++++----------------------------- db-sync.orig | 191 ---------------------------------------- 2 files changed, 144 insertions(+), 328 deletions(-) delete mode 100755 db-sync.orig diff --git a/db-sync b/db-sync index 81dee24..c90f89b 100755 --- a/db-sync +++ b/db-sync @@ -13,174 +13,181 @@ # TODO # * make a tarball of files used for forensics -# * get files db # Run as `V=true db-sync` to get verbose output VERBOSE=${V} ${VERBOSE} && extra="-v" +WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") +trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT + # Returns contents of a repo get_repos() { - mkdir -p ${TMPDIR}/$0.$$.cache -# Exclude everything but db files - rsync ${extra} -mrtlH --no-p --include="*/" \ - --include="*.db" \ - --include="*${DBEXT}" \ - --exclude="*" \ - --delete-after \ - rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/$0.$$.cache + # Exclude everything but db files + rsync ${extra} --no-motd -mrtlH --no-p --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --include="*.files" \ + --include="*${FILESEXT}" \ + --exclude="*" \ + --delete-after \ + rsync://${mirror}/${mirrorpath}/ "$WORKDIR" } get_repo_content() { -# Return all contents - bsdtar tf ${1} | \ - cut -d "/" -f 1 | \ - sort -u + # Return all contents + bsdtar tf ${1} | \ + cut -d "/" -f 1 | \ + sort -u } # Prints blacklisted packages get_blacklist() { - cut -d ':' -f 1 "${BLACKLIST_FILE}" + cut -d ':' -f 1 "${BLACKLIST_FILE}" } # repo # arch get_repo_file() { -# shopt -s nullglob - - echo "${TMPDIR}/$0.$$.cache/${1}/os/${2}/${1}${DBEXT}" + echo "${WORKDIR}/${1}/os/${2}/${1}" } # Process the databases and get the libre packages init() { -# Get the blacklisted packages - blacklist=($(get_blacklist)) -# Store all the whitelist files - whitelists=() - - msg "${#blacklist[@]} packages in blacklist" - -# Sync the repos databases - get_repos - -# Traverse all repo-arch pairs - for _repo in ${ARCHREPOS[@]}; do - for _arch in ${ARCHARCHES[@]}; do - msg "Processing ${_repo}-${_arch}" - - repo_file=$(get_repo_file ${_repo} ${_arch}) - - if [ ! -f "${repo_file}" ]; then - warning "${repo_file} doesn't exist, skipping this repo-arch" - continue - fi - -# Remove blacklisted packages and count them -# TODO capture all removed packages for printing on debug mode - msg2 "Removing blacklisted packages: $( - LC_ALL=C repo-remove ${repo_file} ${blacklist[@]} 2>&1 | \ - grep "\-> Removing" 2>/dev/null| wc -l)" - -# Get db contents - db=($(get_repo_content ${repo_file})) - - msg2 "Process clean db for syncing..." - -# Create a whitelist, add * wildcard to end -# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if -# we aren't syncing them - echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist - - msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" - -# Sync excluding everything but whitelist -# We delete here for cleanup - rsync ${extra} -rtlH \ - --delete-after \ - --delete-excluded \ - --delay-updates \ - --include-from=/tmp/${_repo}-${_arch}.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ - -# Add a new whitelist - whitelists+=(/tmp/${_repo}-${_arch}.whitelist) - - msg "Putting databases back in place" - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - ${TMPDIR}/$0.$$.cache/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ - -# Cleanup - unset db - done - done - - - msg "Syncing package pool" -# Concatenate all whitelists - cat ${whitelists[@]} | sort -u > /tmp/any.whitelist - - msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" - -# Sync -# *Don't delete-after*, this is the job of cleanup scripts. It will remove our -# packages too - for PKGPOOL in ${PKGPOOLS[@]}; do - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ - ${FTP_BASE}/${PKGPOOL}/ - done - -# Sync sources - msg "Syncing source pool" - #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist - - #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool" -# Sync -# *Don't delete-after*, this is the job of cleanup scripts. It will remove our -# packages too - for SRCPOOL in ${SRCPOOLS[@]}; do - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ - ${FTP_BASE}/${SRCPOOL}/ - done - - -# Cleanup - unset blacklist whitelists _arch _repo repo_file + # Get the blacklisted packages + blacklist=($(get_blacklist)) + # Store all the whitelist files + whitelists=() + + msg "%d packages in blacklist" ${#blacklist[@]} + + # Sync the repos databases + get_repos + + # Traverse all repo-arch pairs + for _repo in ${ARCHREPOS[@]}; do + for _arch in ${ARCHARCHES[@]}; do + msg "Processing ${_repo}-${_arch}" + + db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT} + files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT} + + if [ ! -f "${db_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${db_file}" + continue + fi + if [ ! -f "${files_file}" ]; then + warning "%s doesn't exist, skipping this repo-arch" "${files_file}" + continue + fi + + # Remove blacklisted packages and count them + # TODO capture all removed packages for printing on debug mode + msg2 "Removing blacklisted packages from %s database..." .db + LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \ + |& sed -n 's/-> Removing/ &/p' + msg2 "Removing blacklisted packages from %s database..." .files + LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \ + |& sed -n 's/-> Removing/ &/p' + # Get db contents + db=($(get_repo_content ${db_file})) + + msg2 "Process clean db for syncing..." + + # Create a whitelist, add * wildcard to end + # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if + # we aren't syncing them + printf '%s\n' "${db[@]}" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist + + msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" + + # Sync excluding everything but whitelist + # We delete here for cleanup + rsync ${extra} --no-motd -rtlH \ + --delete-after \ + --delete-excluded \ + --delay-updates \ + --include-from=/tmp/${_repo}-${_arch}.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + + # Add a new whitelist + whitelists+=(/tmp/${_repo}-${_arch}.whitelist) + + msg "Putting databases back in place" + rsync ${extra} --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + ${WORKDIR}/${_repo}/os/${_arch}/ \ + ${FTP_BASE}/${_repo}/os/${_arch}/ + + # Cleanup + unset db + done + done + + + msg "Syncing package pool" + # Concatenate all whitelists + cat ${whitelists[@]} | sort -u > /tmp/any.whitelist + + msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" + + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + for PKGPOOL in ${PKGPOOLS[@]}; do + rsync ${extra} --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ + ${FTP_BASE}/${PKGPOOL}/ + done + + # Sync sources + msg "Syncing source pool" + #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist + #msg2 "Retrieving %d sources from pool" $(wc -l < /tmp/any-src.whitelist) + + # Sync + # *Don't delete-after*, this is the job of cleanup scripts. It will remove our + # packages too + for SRCPOOL in ${SRCPOOLS[@]}; do + rsync ${extra} --no-motd -rtlH \ + --delay-updates \ + --safe-links \ + --include-from=/tmp/any.whitelist \ + --exclude="*" \ + rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ + ${FTP_BASE}/${SRCPOOL}/ + done + + # Cleanup + unset blacklist whitelists _arch _repo repo_file } trap_exit() { - echo - error "$@" - exit 1 + local signal=$1; shift + echo + error "$@" + trap -- "$signal" + kill "-$signal" "$$" } - -source $(dirname $0)/config -source $(dirname $0)/local_config -source $(dirname $0)/libremessages +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/local_config" +source "$(dirname "$(readlink -e "$0")")/libremessages" # From makepkg set -E - -trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT -trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT -trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR +for signal in TERM HUP QUIT; do + trap "trap_exit $signal '%s signal caught. Exiting...' $signal" $signal +done +trap 'trap_exit INT "Aborted by user! Exiting..."' INT +trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR init - -rm -r ${TMPDIR}/$0.$$.cache diff --git a/db-sync.orig b/db-sync.orig deleted file mode 100755 index 0e07757..0000000 --- a/db-sync.orig +++ /dev/null @@ -1,191 +0,0 @@ -#!/bin/bash -# Syncs Arch repos based on info contained in repo.db files -# License: GPLv3 - -# Principles -# * Get repo.db from an Arch-like repo -# * Generate a list of available packages -# * Create sync whitelist (based on package blacklist) -# * Get packages -# * Check package signatures -# * Check database signatures -# * Sync repo => repo - -# TODO -# * make a tarball of files used for forensics -# * get files db - -# Run as `V=true db-sync` to get verbose output -VERBOSE=${V} -${VERBOSE} && extra="-v" - -# Returns contents of a repo -get_repos() { - mkdir -p ${TMPDIR}/${0##*/}.$$.cache - # Exclude everything but db files - rsync ${extra} -mrtlH --no-p --include="*/" \ - --include="*.db" \ - --include="*${DBEXT}" \ - --include="*.files" \ - --include="*${FILESEXT}" \ - --exclude="*" \ - --delete-after \ - rsync://${mirror}/${mirrorpath}/ ${TMPDIR}/${0##*/}.$$.cache -} - -get_repo_content() { - # Return all contents - bsdtar tf ${1} | \ - cut -d "/" -f 1 | \ - sort -u -} - -# Prints blacklisted packages -get_blacklist() { - cut -d ':' -f 1 "${BLACKLIST_FILE}" -} - -# repo -# arch -get_repo_file() { - echo "${TMPDIR}/${0##*/}.$$.cache/${1}/os/${2}/${1}" -} - -# Process the databases and get the libre packages -init() { - - # Get the blacklisted packages - blacklist=($(get_blacklist)) - # Store all the whitelist files - whitelists=() - - msg "${#blacklist[@]} packages in blacklist" - - # Sync the repos databases - get_repos - - # Traverse all repo-arch pairs - for _repo in ${ARCHREPOS[@]}; do - for _arch in ${ARCHARCHES[@]}; do - msg "Processing ${_repo}-${_arch}" - - db_file=$(get_repo_file ${_repo} ${_arch})${DBEXT} - files_file=$(get_repo_file ${_repo} ${_arch})${FILESEXT} - - if [ ! -f "${db_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${db_file}" - continue - fi - if [ ! -f "${files_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${files_file}" - continue - fi - - # Remove blacklisted packages and count them - # TODO capture all removed packages for printing on debug mode - msg2 "Removing blacklisted packages from .db database..." - LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" - msg2 "Removing blacklisted packages from .files database..." - LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" - - # Get db contents - db=($(get_repo_content ${db_file})) - - msg2 "Process clean db for syncing..." - - # Create a whitelist, add * wildcard to end - # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if - # we aren't syncing them - echo ${db[@]} | tr ' ' "\n" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist - - msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" - - # Sync excluding everything but whitelist - # We delete here for cleanup - rsync ${extra} -rtlH \ - --delete-after \ - --delete-excluded \ - --delay-updates \ - --include-from=/tmp/${_repo}-${_arch}.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ - - # Add a new whitelist - whitelists+=(/tmp/${_repo}-${_arch}.whitelist) - - msg "Putting databases back in place" - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - ${TMPDIR}/${0##*/}.$$.cache/${_repo}/os/${_arch}/ \ - ${FTP_BASE}/${_repo}/os/${_arch}/ - - # Cleanup - unset db - done - done - - - msg "Syncing package pool" - # Concatenate all whitelists - cat ${whitelists[@]} | sort -u > /tmp/any.whitelist - - msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" - - # Sync - # *Don't delete-after*, this is the job of cleanup scripts. It will remove our - # packages too - for PKGPOOL in ${PKGPOOLS[@]}; do - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ - ${FTP_BASE}/${PKGPOOL}/ - done - - # Sync sources - msg "Syncing source pool" - #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist - - #msg2 "Retrieving $(wc -l /tmp/any-src.whitelist | cut -d' ' -f1) sources from pool" - # Sync - # *Don't delete-after*, this is the job of cleanup scripts. It will remove our - # packages too - for SRCPOOL in ${SRCPOOLS[@]}; do - rsync ${extra} -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ - ${FTP_BASE}/${SRCPOOL}/ - done - - # Cleanup - unset blacklist whitelists _arch _repo repo_file -} - -trap_exit() { - echo - error "$@" - exit 1 -} - - -source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/local_config" -source "$(dirname "$(readlink -e "$0")")/libremessages" - -# From makepkg -set -E - -trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT -trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT -trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR - -init - -rm -r ${TMPDIR}/${0##*/}.$$.cache -- cgit v1.2.3-2-g168b From 0b52c6e030fc60377a4dfc0a4e142bee50b0344e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Wed, 29 Jan 2014 19:01:46 -0200 Subject: add [multilib-testing], [libre-multilib] and [libre-multilib-testing] repos --- config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config b/config index c3f9a55..29c42fe 100644 --- a/config +++ b/config @@ -3,9 +3,9 @@ FTP_BASE="/srv/http/repo/public" SVNREPO="/var/abs" # Repos from Arch -ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib') +ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib' 'multilib-testing') # Official Parabola repos -OURREPOS=('libre' 'libre-testing') +OURREPOS=('libre' 'libre-testing' 'libre-multilib' 'libre-multilib-testing') # User repos USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan') # Community project repos -- cgit v1.2.3-2-g168b From 5bbd4077065e9a10bab0c709c9e3edb0b7d0fc05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 12 Mar 2014 10:46:56 -0300 Subject: Hopefully fix a horrible bug that caused leaked packages Also added a few checks --- db-sync | 24 ++++++++++++++++++------ libremessages | 6 ++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/db-sync b/db-sync index c90f89b..4a28aa1 100755 --- a/db-sync +++ b/db-sync @@ -62,6 +62,8 @@ init() { msg "%d packages in blacklist" ${#blacklist[@]} + test ${#blacklist[@]} -eq 0 && fatal_error "Empty blacklist" + # Sync the repos databases get_repos @@ -86,10 +88,10 @@ init() { # TODO capture all removed packages for printing on debug mode msg2 "Removing blacklisted packages from %s database..." .db LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \ - |& sed -n 's/-> Removing/ &/p' + |& sed -n 's/-> Removing/ &/p' msg2 "Removing blacklisted packages from %s database..." .files LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \ - |& sed -n 's/-> Removing/ &/p' + |& sed -n 's/-> Removing/ &/p' # Get db contents db=($(get_repo_content ${db_file})) @@ -98,7 +100,10 @@ init() { # Create a whitelist, add * wildcard to end # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if # we aren't syncing them - printf '%s\n' "${db[@]}" | sed "s|$|*|g" > /tmp/${_repo}-${_arch}.whitelist + # IMPORTANT: the . in the sed command is needed because an empty + # whitelist would consist of a single * allowing any package to + # pass through + printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > /tmp/${_repo}-${_arch}.whitelist msg2 "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1) packages in whitelist" @@ -130,8 +135,8 @@ init() { msg "Syncing package pool" - # Concatenate all whitelists - cat ${whitelists[@]} | sort -u > /tmp/any.whitelist + # Concatenate all whitelists, check for single *s just in case + cat ${whitelists[@]} | grep -v "^\*$" | sort -u > /tmp/any.whitelist msg2 "Retrieving $(wc -l /tmp/any.whitelist | cut -d' ' -f1) packages from pool" @@ -182,10 +187,17 @@ source "$(dirname "$(readlink -e "$0")")/config" source "$(dirname "$(readlink -e "$0")")/local_config" source "$(dirname "$(readlink -e "$0")")/libremessages" +# Check variables presence +for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE + FTP_BASE SRCPOOLS PKGPOOLS; do + + test -z "${!var}" && fatal_error "Empty ${var}" +done + # From makepkg set -E for signal in TERM HUP QUIT; do - trap "trap_exit $signal '%s signal caught. Exiting...' $signal" $signal + trap "trap_exit $signal '%s signal caught. Exiting...' $signal" $signal done trap 'trap_exit INT "Aborted by user! Exiting..."' INT trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR diff --git a/libremessages b/libremessages index 9fbbc2b..0c6ded1 100755 --- a/libremessages +++ b/libremessages @@ -75,3 +75,9 @@ error() { printf "${RED}==> $(gettext "ERROR:")${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 } +fatal_error() { + local mesg=$1; shift + error "$mesg" "$@" + + exit 1 +} -- cgit v1.2.3-2-g168b From dc7260243263b8e49bbc308eb67945c43f3bf122 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 12 Mar 2014 10:48:26 -0300 Subject: oops --- db-sync | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/db-sync b/db-sync index 4a28aa1..58bb465 100755 --- a/db-sync +++ b/db-sync @@ -188,8 +188,7 @@ source "$(dirname "$(readlink -e "$0")")/local_config" source "$(dirname "$(readlink -e "$0")")/libremessages" # Check variables presence -for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE - FTP_BASE SRCPOOLS PKGPOOLS; do +for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE SRCPOOLS PKGPOOLS; do test -z "${!var}" && fatal_error "Empty ${var}" done -- cgit v1.2.3-2-g168b From d902025a8a000c1383c7612e9f442c6734290169 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 15 Jun 2014 22:20:18 +0000 Subject: put our packages in a separate pool --- config | 12 ++++++++---- db-sync | 16 +++++++++------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/config b/config index 29c42fe..c6efac6 100644 --- a/config +++ b/config @@ -12,14 +12,18 @@ USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jor PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'java-ugly' 'nonprism') # Remote repos PKGREPOS=("${ARCHREPOS[@]}" "${OURREPOS[@]}" "${USERREPOS[@]}" "${PROJREPOS[@]}") -PKGPOOL='pool/packages' -SRCPOOL='sources/packages' +PKGPOOL='pool/parabola' +SRCPOOL='sources/parabola' # Directories where packages are shared between repos # *relative to FTP_BASE* -PKGPOOLS=('pool/packages' 'pool/community') +ARCHPKGPOOLS=(pool/{packages,community}) +OURPKGPOOLS=(pool/parabola) +PKGPOOLS=(${OURPKGPOOLS[@]} ${ARCHPKGPOOLS[@]}) # Directories where sources are stored -SRCPOOLS=('sources/packages' 'sources/community') +ARCHSRCPOOLS=(sources/{packages,community}) +OURPKGPOOLS=(sources/parabola) +SRCPOOLS=(${OURSRCPOOLS[@]} ${ARCHSRCPOOLS[@]}) CLEANUP_DESTDIR="$FTP_BASE/old/packages" CLEANUP_DRYRUN=false diff --git a/db-sync b/db-sync index 58bb465..4e692e9 100755 --- a/db-sync +++ b/db-sync @@ -143,14 +143,15 @@ init() { # Sync # *Don't delete-after*, this is the job of cleanup scripts. It will remove our # packages too - for PKGPOOL in ${PKGPOOLS[@]}; do + local pkgpool + for pkgpool in ${ARCHPKGPOOLS[@]}; do rsync ${extra} --no-motd -rtlH \ --delay-updates \ --safe-links \ --include-from=/tmp/any.whitelist \ --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${PKGPOOL}/ \ - ${FTP_BASE}/${PKGPOOL}/ + rsync://${mirror}/${mirrorpath}/${pkgpool}/ \ + ${FTP_BASE}/${pkgpool}/ done # Sync sources @@ -161,14 +162,15 @@ init() { # Sync # *Don't delete-after*, this is the job of cleanup scripts. It will remove our # packages too - for SRCPOOL in ${SRCPOOLS[@]}; do + local srcpool + for srcpool in ${ARCHSRCPOOLS[@]}; do rsync ${extra} --no-motd -rtlH \ --delay-updates \ --safe-links \ --include-from=/tmp/any.whitelist \ --exclude="*" \ - rsync://${mirror}/${mirrorpath}/${SRCPOOL}/ \ - ${FTP_BASE}/${SRCPOOL}/ + rsync://${mirror}/${mirrorpath}/${srcpool}/ \ + ${FTP_BASE}/${srcpool}/ done # Cleanup @@ -188,7 +190,7 @@ source "$(dirname "$(readlink -e "$0")")/local_config" source "$(dirname "$(readlink -e "$0")")/libremessages" # Check variables presence -for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE SRCPOOLS PKGPOOLS; do +for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do test -z "${!var}" && fatal_error "Empty ${var}" done -- cgit v1.2.3-2-g168b From d8e9a3ea7c9d8924eef16e6bd9725cf151151936 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 15 Jun 2014 23:02:17 -0400 Subject: config: don't clobber the existing STAGING scheme --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index baf5da7..63698b9 100644 --- a/config +++ b/config @@ -41,7 +41,7 @@ REQUIRE_SIGNATURE=true LOCK_DELAY=10 LOCK_TIMEOUT=300 -[ -n "${STAGING:-}" ] || STAGING="$HOME/staging" +[ -n "${STAGING:-}" ] || STAGING="$HOME/staging/unknown/staging" TMPDIR="/tmp" ARCHARCHES=(i686 x86_64) OURARCHES=(mips64el) -- cgit v1.2.3-2-g168b From fae18d372c7d5fad545526b554e9cde657192226 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 16 Jun 2014 10:59:34 -0400 Subject: config:remove java-ugly --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index c6efac6..c7ea7c2 100644 --- a/config +++ b/config @@ -9,7 +9,7 @@ OURREPOS=('libre' 'libre-testing' 'libre-multilib' 'libre-multilib-testing') # User repos USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan') # Community project repos -PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'java-ugly' 'nonprism') +PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'nonprism') # Remote repos PKGREPOS=("${ARCHREPOS[@]}" "${OURREPOS[@]}" "${USERREPOS[@]}" "${PROJREPOS[@]}") PKGPOOL='pool/parabola' -- cgit v1.2.3-2-g168b From 7a94b2db24e707606d9e620ba1b8a9462dce5e0f Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 16 Jun 2014 11:01:06 -0400 Subject: config: don't have default STAGING be public --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index c7ea7c2..75e88cb 100644 --- a/config +++ b/config @@ -40,7 +40,7 @@ REQUIRE_SIGNATURE=true LOCK_DELAY=10 LOCK_TIMEOUT=300 -[ -n "${STAGING:-}" ] || STAGING="$FTP_BASE/staging" +[ -n "${STAGING:-}" ] || STAGING="$HOME/staging/unknown/staging" TMPDIR="/tmp" ARCHARCHES=(i686 x86_64) OURARCHES=(mips64el) -- cgit v1.2.3-2-g168b From 73dcd17cf3201295f2ee29f63b4353a5e3c95779 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 16 Jun 2014 12:16:56 -0400 Subject: abslibre: also run git gc --- abslibre | 1 + 1 file changed, 1 insertion(+) diff --git a/abslibre b/abslibre index a5733f0..09105c1 100755 --- a/abslibre +++ b/abslibre @@ -90,6 +90,7 @@ sync_pre_mips64el() { git add . && \ git commit -m \"$(date)\" -a git push origin master + git gc " } -- cgit v1.2.3-2-g168b From 64a9e2fcdeaf99ae982270743c6cca4dd5314b5c Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 16 Jun 2014 12:20:45 -0400 Subject: abslibre: use tab indent --- abslibre | 111 +++++++++++++++++++++++++++++++-------------------------------- 1 file changed, 55 insertions(+), 56 deletions(-) diff --git a/abslibre b/abslibre index 09105c1..0b3e371 100755 --- a/abslibre +++ b/abslibre @@ -23,44 +23,44 @@ REPOS=(core extra community testing community-testing !staging !community-stagin # * Create repo.abs.tar.gz tarballs function sync_abs() { - for ARCH in any i686 x86_64; do - rsync ${SYNCARGS} ${SYNCSERVER}::abs/${ARCH}/ ${ABSROOT}/${ARCH} || return $? - done + for ARCH in any i686 x86_64; do + rsync ${SYNCARGS} ${SYNCSERVER}::abs/${ARCH}/ ${ABSROOT}/${ARCH} || return $? + done } function get_blacklist() { printf ":: Updating blacklist...\t" wget -q -O - "${BLACKLIST}" | cut -d':' -f1 | sort -u | \ - sed "s/^/**\//" > ${BLFILE} || { + sed "s/^/**\//" > ${BLFILE} || { printf "[FAILED]\n" return 1 } -# Prevent using an empty blacklist - [ $(wc -l ${BLFILE} | cut -d " " -f1) -eq 0 ] && return 1 + # Prevent using an empty blacklist + [ $(wc -l ${BLFILE} | cut -d " " -f1) -eq 0 ] && return 1 printf "[OK]\n" } function sync_abs_libre() { -# Clone ABSLibre git repo - if [ -d /var/tmp/abslibre/.git ]; then - pushd /var/tmp/abslibre >/dev/null 2>&1 - git pull - popd >/dev/null 2>&1 - else - git clone /srv/git/abslibre.git /var/tmp/abslibre - fi + # Clone ABSLibre git repo + if [ -d /var/tmp/abslibre/.git ]; then + pushd /var/tmp/abslibre >/dev/null 2>&1 + git pull + popd >/dev/null 2>&1 + else + git clone /srv/git/abslibre.git /var/tmp/abslibre + fi -# Sync from ABS and then sync from ABSLibre + # Sync from ABS and then sync from ABSLibre printf ":: Syncing ABSLibre...\t" (rsync ${SYNCARGS} --delete-excluded \ - --exclude-from=${BLFILE} \ - ${ABSROOT} \ - ${ABSLIBRE} \ - && - for ARCH in i686 x86_64; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --exclude=.git/ /var/tmp/abslibre/ ${ABSLIBRE}/${ARCH}/; done) || { + --exclude-from=${BLFILE} \ + ${ABSROOT} \ + ${ABSLIBRE} \ + && + for ARCH in i686 x86_64; do rsync -v -mrtq --no-motd --no-p --no-o --no-g --exclude=.git/ /var/tmp/abslibre/ ${ABSLIBRE}/${ARCH}/; done) || { printf "[FAILED]\n" return 1 } @@ -70,46 +70,45 @@ function sync_abs_libre() { # This part is very hacky and particular to the current setup :P sync_pre_mips64el() { - pushd /home/fauno/Repos/abslibre-pre-mips64el >/dev/null - - sudo -u fauno sh -c " - - rsync ${SYNCARGS} \ - --exclude=.git* \ - --exclude=community-staging \ - --exclude=community-testing \ - --exclude=gnome-unstable \ - --exclude=kde-unstable \ - --exclude=multilib \ - --exclude=multilib-testing \ - --exclude=multilib-staging \ - --exclude=staging \ - --exclude=testing \ - ${ABSLIBRE}/x86_64/ \ - /home/fauno/Repos/abslibre-pre-mips64el/ && \ - git add . && \ - git commit -m \"$(date)\" -a - git push origin master - git gc - " + pushd /home/fauno/Repos/abslibre-pre-mips64el >/dev/null + + sudo -u fauno sh -c " + rsync ${SYNCARGS} \ + --exclude=.git* \ + --exclude=community-staging \ + --exclude=community-testing \ + --exclude=gnome-unstable \ + --exclude=kde-unstable \ + --exclude=multilib \ + --exclude=multilib-testing \ + --exclude=multilib-staging \ + --exclude=staging \ + --exclude=testing \ + ${ABSLIBRE}/x86_64/ \ + /home/fauno/Repos/abslibre-pre-mips64el/ && + git add . && + git commit -m \"$(date)\" -a + git push origin master + git gc + " } # Create .abs.tar.gz tarballs create_tarballs() { - for repo in ${ABSLIBRE}/{i686,x86_64}/*; do - baserepo=${repo##*/} - arch=$(basename $(dirname $repo)) - -# Remove the old one - mkdir -p /srv/http/web/media/abs/$baserepo/os/$arch/ - rm /srv/http/web/media/abs/$baserepo/os/$arch/$baserepo.abs.tar.gz -# Create a new one joining arch and any -# Remove the first part of the path (it could be $repo but any isn't hit) - bsdtar -czvf /srv/http/web/media/abs/$baserepo/os/$arch/$baserepo.abs.tar.gz \ - -s ":${ABSLIBRE}/[a-z0-9_]\+/[a-z]\+::" \ - $repo/* ${ABSLIBRE}/any/${baserepo}/* - - done + for repo in ${ABSLIBRE}/{i686,x86_64}/*; do + baserepo=${repo##*/} + arch=$(basename $(dirname $repo)) + + # Remove the old one + mkdir -p /srv/http/web/media/abs/$baserepo/os/$arch/ + rm /srv/http/web/media/abs/$baserepo/os/$arch/$baserepo.abs.tar.gz + # Create a new one joining arch and any + # Remove the first part of the path (it could be $repo but any isn't hit) + bsdtar -czvf /srv/http/web/media/abs/$baserepo/os/$arch/$baserepo.abs.tar.gz \ + -s ":${ABSLIBRE}/[a-z0-9_]\+/[a-z]\+::" \ + $repo/* ${ABSLIBRE}/any/${baserepo}/* + + done } sync_abs || exit 1 -- cgit v1.2.3-2-g168b