From 909b017c08109bda405a3e38a59cbf26211e6683 Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Fri, 29 Aug 2008 20:23:26 +0200 Subject: Replace check_archlinux.py by check_archlinux/check_packages.py The old script had several problems so I decided to do a full rewrite. The improvements include : * better and safer parsing of PKGBUILDs It now uses separate parse_pkgbuilds.sh bash script (inspired from namcap) * much better performance A python module for calling vercmp natively, and the algorithm for checking circular dependencies was greatly improved * more accurate dependency and provision handling Now versioned dependencies and provisions are handled correctly. After building the python module and moving it next to the main script, it should be possible to use it like this : For core and extra : ./check_packages.py --abs-tree=/home/abs/rsync/i686 --repos=core,extra ./check_packages.py --abs-tree=/home/abs/rsync/x86_64 --repos=core,extra For community : ./check_packages.py --abs-tree=/home/abs/rsync/i686 --repos=community ./check_packages.py --abs-tree=/home/abs/rsync/x86_64 --repos=community Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 78 ++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100755 cron-jobs/check_archlinux/parse_pkgbuilds.sh (limited to 'cron-jobs/check_archlinux/parse_pkgbuilds.sh') diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh new file mode 100755 index 0000000..d4205ae --- /dev/null +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +parse() { + unset pkgname pkgver pkgrel + unset depends makedepends conflicts provides + ret=0 + dir=$1 + pkgbuild=$dir/PKGBUILD + source $pkgbuild &>/dev/null || ret=$? + + # ensure $pkgname and $pkgver variables were found + if [ $ret -ne 0 -o -z "$pkgname" -o -z "$pkgver" ]; then + echo -e "%INVALID%\n$pkgbuild\n" + return 1 + fi + + echo -e "%NAME%\n$pkgname\n" + echo -e "%VERSION%\n$pkgver-$pkgrel\n" + echo -e "%PATH%\n$dir\n" + + if [ -n "$arch" ]; then + echo "%ARCH%" + for i in ${arch[@]}; do echo $i; done + echo "" + fi + if [ -n "$depends" ]; then + echo "%DEPENDS%" + for i in ${depends[@]}; do + echo $i + done + echo "" + fi + if [ -n "$makedepends" ]; then + echo "%MAKEDEPENDS%" + for i in ${makedepends[@]}; do + echo $i + done + echo "" + fi + if [ -n "$conflicts" ]; then + echo "%CONFLICTS%" + for i in ${conflicts[@]}; do echo $i; done + echo "" + fi + if [ -n "$provides" ]; then + echo "%PROVIDES%" + for i in ${provides[@]}; do echo $i; done + echo "" + fi + return 0 +} + +find_pkgbuilds() { + if [ -f $1/PKGBUILD ]; then + parse $1 + return + fi + empty=1 + for dir in $1/*; do + if [ -d $dir ]; then + find_pkgbuilds $dir + unset empty + fi + done + if [ -n "$empty" ]; then + echo -e "%MISSING%\n$1\n" + fi +} + +if [ -z "$*" ]; then + exit 1 +fi + +for dir in "$@"; do + find_pkgbuilds $dir +done + +exit 0 -- cgit v1.2.3-2-g168b From d39b68ede8567fb2ee15f45995ddab8f4710e9dc Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Sat, 3 Jan 2009 13:22:57 +0100 Subject: check_packages : add --arch option. The parsing script didn't set CARCH previously, and the flashplugin PKGBUILD exited in this case. First override the exit function to prevent the whole script to exit, and add a --arch option to be able to set CARCH correctly. To be used like this : For core and extra : ./check_packages.py --abs-tree=/home/abs/rsync/i686 --repos=core,extra --arch=i686 ./check_packages.py --abs-tree=/home/abs/rsync/x86_64 --repos=core,extra --arch=x86_64 For community : ./check_packages.py --abs-tree=/home/abs/rsync/i686 --repos=community --arch=i686 ./check_packages.py --abs-tree=/home/abs/rsync/x86_64 --repos=community --arch=x86_64 Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'cron-jobs/check_archlinux/parse_pkgbuilds.sh') diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index d4205ae..47aec89 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -1,5 +1,10 @@ #!/bin/bash +# Usage : parse_pkgbuilds.sh arch +# Example : parse_pkgbuilds.sh i686 /var/abs/core /var/abs/extra + +exit() { return; } + parse() { unset pkgname pkgver pkgrel unset depends makedepends conflicts provides @@ -67,10 +72,12 @@ find_pkgbuilds() { fi } -if [ -z "$*" ]; then +if [ -z "$1" -o -z "$2" ]; then exit 1 fi +CARCH=$1 +shift for dir in "$@"; do find_pkgbuilds $dir done -- cgit v1.2.3-2-g168b From f515029a694a11065f95a79bd83b325bf04562bb Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 8 May 2009 13:21:39 -0700 Subject: check_archlinux: Skip searching CVS and .svn dirs Prevents some errors in the community scripts due to scanning of CVS dirs. Also skipping .svn dirs for the future (dotglob may be set, in which case we'd scan .svn dirs as well) Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'cron-jobs/check_archlinux/parse_pkgbuilds.sh') diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index 47aec89..7dcfbbb 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -56,6 +56,12 @@ parse() { } find_pkgbuilds() { + #Skip over some dirs + local d="$(basename $1)" + if [ "$d" = "CVS" -o "$d" = ".svn" ]; then + return + fi + if [ -f $1/PKGBUILD ]; then parse $1 return -- cgit v1.2.3-2-g168b From d455a0b2cd72cd08b7b6cde878a29841f77a041e Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Fri, 28 Aug 2009 01:27:11 +0200 Subject: check_packages : add support for split packages. I just found a way to support split packages, by using $(type package_${pkg}), parsing that output and running eval on the relevant lines. This is a bit ugly, and while it works fine on my machine and my current abs tree, I cannot guarantee this code is bug free :) Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 82 +++++++++++++++++++++++----- 1 file changed, 69 insertions(+), 13 deletions(-) (limited to 'cron-jobs/check_archlinux/parse_pkgbuilds.sh') diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index 7dcfbbb..0faa29f 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -5,24 +5,36 @@ exit() { return; } -parse() { - unset pkgname pkgver pkgrel - unset depends makedepends conflicts provides - ret=0 - dir=$1 - pkgbuild=$dir/PKGBUILD - source $pkgbuild &>/dev/null || ret=$? +variables=('pkgname' 'pkgver' 'pkgrel' 'depends' 'makedepends' 'provides' 'conflicts' ) +readonly -a variables - # ensure $pkgname and $pkgver variables were found - if [ $ret -ne 0 -o -z "$pkgname" -o -z "$pkgver" ]; then - echo -e "%INVALID%\n$pkgbuild\n" - return 1 - fi +backup_package_variables() { + for var in ${variables[@]}; do + indirect="${var}_backup" + eval "${indirect}=(\${$var[@]})" + done +} + +restore_package_variables() { + for var in ${variables[@]}; do + indirect="${var}_backup" + if [ -n "${!indirect}" ]; then + eval "${var}=(\${$indirect[@]})" + else + unset ${var} + fi + done +} +print_info() { echo -e "%NAME%\n$pkgname\n" echo -e "%VERSION%\n$pkgver-$pkgrel\n" echo -e "%PATH%\n$dir\n" + if [ -n "$pkgbase" ]; then + echo -e "%BASE%\n$pkgbase\n" + fi + if [ -n "$arch" ]; then echo "%ARCH%" for i in ${arch[@]}; do echo $i; done @@ -52,6 +64,50 @@ parse() { for i in ${provides[@]}; do echo $i; done echo "" fi +} + +source_pkgbuild() { + ret=0 + dir=$1 + pkgbuild=$dir/PKGBUILD + for var in ${variables[@]}; do + unset ${var} + done + source $pkgbuild &>/dev/null || ret=$? + + # ensure $pkgname and $pkgver variables were found + if [ $ret -ne 0 -o -z "$pkgname" -o -z "$pkgver" ]; then + echo -e "%INVALID%\n$pkgbuild\n" + return 1 + fi + + if [ "${#pkgname[@]}" -gt "1" ]; then + for pkg in ${pkgname[@]}; do + if [ "$(type -t package_${pkg})" != "function" ]; then + echo -e "%INVALID%\n$pkgbuild\n" + return 1 + else + backup_package_variables + pkgname=$pkg + while IFS= read -r line; do + var=${line%%=*} + var="${var#"${var%%[![:space:]]*}"}" # remove leading whitespace characters + for realvar in ${variables[@]}; do + if [ "$var" == "$realvar" ]; then + eval $line + break + fi + done + done < <(type package_${pkg}) + print_info + restore_package_variables + fi + done + else + echo + print_info + fi + return 0 } @@ -63,7 +119,7 @@ find_pkgbuilds() { fi if [ -f $1/PKGBUILD ]; then - parse $1 + source_pkgbuild $1 return fi empty=1 -- cgit v1.2.3-2-g168b From b36b754497df69778273f6a6e7b0d91679646565 Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Thu, 3 Sep 2009 23:34:24 +0200 Subject: parse_pkgbuilds : simple performance tweak this simple tweak gives a nice perf boost : from 10s to 7s to parse extra repo. indeed calling basename caused a fork to happen for the thousand of files being considered Now the major bottleneck is parsing split pkgbuilds, but I am afraid there is no magical solution for that :( Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs/check_archlinux/parse_pkgbuilds.sh') diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index 0faa29f..c2df7d1 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -113,7 +113,7 @@ source_pkgbuild() { find_pkgbuilds() { #Skip over some dirs - local d="$(basename $1)" + local d="${1##*/}" if [ "$d" = "CVS" -o "$d" = ".svn" ]; then return fi -- cgit v1.2.3-2-g168b From 12aba89ed2100552ddfe7d69bbd9e35a2019d098 Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Tue, 15 Sep 2009 09:19:02 +0200 Subject: parse_pkgbuilds.sh : fix pkgbase handling add the default value for pkgbase also rework slightly how variables are handled (better distinction between splitpkg variables and the rest) Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'cron-jobs/check_archlinux/parse_pkgbuilds.sh') diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index c2df7d1..5cd17e4 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -5,18 +5,19 @@ exit() { return; } -variables=('pkgname' 'pkgver' 'pkgrel' 'depends' 'makedepends' 'provides' 'conflicts' ) -readonly -a variables +splitpkg_overrides=('depends' 'optdepends' 'provides' 'conflicts') +variables=('pkgname' 'pkgbase' 'pkgver' 'pkgrel' 'makedepends' 'arch' ${splitpkg_overrides[@]}) +readonly -a variables splitpkg_overrides backup_package_variables() { - for var in ${variables[@]}; do + for var in ${splitpkg_overrides[@]}; do indirect="${var}_backup" eval "${indirect}=(\${$var[@]})" done } restore_package_variables() { - for var in ${variables[@]}; do + for var in ${splitpkg_overrides[@]}; do indirect="${var}_backup" if [ -n "${!indirect}" ]; then eval "${var}=(\${$indirect[@]})" @@ -82,6 +83,7 @@ source_pkgbuild() { fi if [ "${#pkgname[@]}" -gt "1" ]; then + pkgbase=${pkgbase:-${pkgname[0]}} for pkg in ${pkgname[@]}; do if [ "$(type -t package_${pkg})" != "function" ]; then echo -e "%INVALID%\n$pkgbuild\n" -- cgit v1.2.3-2-g168b