From 511759504993c858d977b649cd02cf7969811b57 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 7 May 2008 02:06:21 -0400 Subject: Moving some files around, organizational Signed-off-by: Aaron Griffin --- cron-jobs/cleanup2.sh | 46 ++++++++++++++++++++++++++++++ cron-jobs/ftpdir-cleanup | 60 +++++++++++++++++++++++++++++++++++++++ cron-jobs/genpkglist | 74 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 180 insertions(+) create mode 100755 cron-jobs/cleanup2.sh create mode 100755 cron-jobs/ftpdir-cleanup create mode 100755 cron-jobs/genpkglist (limited to 'cron-jobs') diff --git a/cron-jobs/cleanup2.sh b/cron-jobs/cleanup2.sh new file mode 100755 index 0000000..4a79654 --- /dev/null +++ b/cron-jobs/cleanup2.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +usage() { + echo "Usage: $0 repo architecture" +} + +getpkgname() { + local tmp + + tmp=${1##*/} + tmp=${tmp%.pkg.tar.gz} + tmp=${tmp%-i686} + tmp=${tmp%-x86_64} + echo ${tmp%-*-*} +} + +FTPBASEDIR="/home/ftp" +FTPDIR=${FTPBASEDIR}/${1}/os/${2} +DBFILE=${FTPDIR}/${1}.db.tar.gz +OBSOLETEFILES="" + +if [ $# -lt 2 -o ! -f ${DBFILE} ]; then + usage + exit 1 +fi + +TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1 +cd ${TMPDIR} +tar xzf ${DBFILE} + +cd ${FTPDIR} +for pkgfile in *.pkg.tar.gz; do + pkgname="$(getpkgname ${pkgfile})" + for p in ${FTPDIR}/${pkgname}-*; do + if [ "$(getpkgname $(basename ${p}))" = "${pkgname}" ]; then + continue 2 + fi + done + OBSOLETEFILES="${OBSOLETEFILES} ${pkgfile}" +done + +cd - >/dev/null +rm -rf ${TMPDIR} + +echo -ne "DIRECTORY:\n${FTPDIR}\n\n" +echo -ne "OBSOLETEFILES:\n${OBSOLETEFILES}\n\n" diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup new file mode 100755 index 0000000..18e26f2 --- /dev/null +++ b/cron-jobs/ftpdir-cleanup @@ -0,0 +1,60 @@ +#!/bin/bash + +usage() { + echo "Usage: $0 repo architecture" +} + +getpkgname() { + local tmp + + tmp=${1##*/} + tmp=${tmp%.pkg.tar.gz} + tmp=${tmp%-i686} + tmp=${tmp%-x86_64} + echo ${tmp%-*-*} +} + +reponame=$1 +arch=$2 + +FTPBASEDIR="/home/ftp" +FTPDIR=${FTPBASEDIR}/$reponame/os/$arch +DBFILE=${FTPDIR}/$reponame.db.tar.gz +MISSINGFILES="" +DELETEFILES="" + +if [ $# -lt 2 -o ! -f ${DBFILE} ]; then + usage + exit 1 +fi + +TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1 + +cd ${TMPDIR} +tar xzf ${DBFILE} +for pkg in *; do + filename=$(grep -A1 '^%FILENAME%$' ${pkg}/desc | tail -n1) + [ -z "${filename}" ] && filename="${pkg}.pkg.tar.gz" + if [ ! -f ${FTPDIR}/${filename} ]; then + MISSINGFILES="${MISSINGFILES} ${filename}" + else + pkgname="$(getpkgname ${filename})" + for otherfile in ${FTPDIR}/${pkgname}-*; do + otherfile="$(basename ${otherfile})" + if [ "${otherfile}" != "${filename}" -a "${pkgname}" = "$(getpkgname ${otherfile})" ]; then + DELETEFILES="${DELETEFILES} ${otherfile}" + fi + done + fi +done + +cd - >/dev/null +rm -rf ${TMPDIR} + +echo -ne "DIRECTORY:\n${FTPDIR}\n\n" +echo -ne "DELETEFILES:\n${DELETEFILES}\n\n" +echo -ne "MISSINGFILES:\n${MISSINGFILES}\n\n" + +cd ${FTPDIR} +[ -n "${DELETEFILES}" ] && mv ${DELETEFILES} /home/package-cleanup/ +cd - >/dev/null diff --git a/cron-jobs/genpkglist b/cron-jobs/genpkglist new file mode 100755 index 0000000..31b7c78 --- /dev/null +++ b/cron-jobs/genpkglist @@ -0,0 +1,74 @@ +#!/bin/bash +# +# genpkglist +# +# Generates a text package database for use with the setup script +# (also used to check for missing packages in the download directory) +# + +# Get the package name from the filename +# hackish, but should work for now +getpkgname() { + local tmp + + tmp=${1##*/} + tmp=${tmp%.pkg.tar.gz} + tmp=${tmp%-i686} + tmp=${tmp%-x86_64} + echo ${tmp%-*-*} +} + +pkgfile="$(pwd)/packages.txt" +ftppath=$1 +repotag=$2 + +rm -f $pkgfile +for package in *; do + if [ -d "$package/$repotag/" ]; then + cd "$package/$repotag/" + if [ -f PKGBUILD ]; then + . PKGBUILD + if [ -f $ftppath/$pkgname-$pkgver-$pkgrel.pkg.tar.gz ]; then + echo "$pkgname-$pkgver-$pkgrel.pkg.tar.gz" >>$pkgfile + elif [ -f $ftppath/$pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz ]; then + echo "$pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz" >>$pkgfile + else + echo "notice: Missing $pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz in ftp site" >&2 + fi + fi + cd .. + fi +done + +cd $ftppath +unset DUPES DUPEFILES last +for pkg in *.pkg.tar.gz; do + pkgname=$(getpkgname $pkg) + if [ "$last" = "$pkgname" ]; then + DUPES="$DUPES $pkgname" + DUPEFILES="$DUPEFILES $pkg" + fi + last=$pkgname +done + +showdupes() { + done= + for i in *.pkg.tar.gz; do + pkgname=$(getpkgname $i) + if [ "$pkgname" = "$1" ]; then + ls -l $i | awk '{print $6" "$7" "$8" "$9}' + done=1 + else + [ "$done" = "1" ] && return + fi + done +} + +if [ "$DUPES" ]; then + DUPES=$((for d in $DUPES; do echo $d; done) | sort -u) + echo "Possible Dupes for $ftppath (remove old versions)" + echo "Date Filename" + for dupe in $DUPES; do + showdupes $dupe + done +fi -- cgit v1.2.3-2-g168b From 39298cf09e2e88b1f6aec779170990675baf8992 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 14 May 2008 18:49:20 -0400 Subject: Cron cleanup, and only copy DB files once This would break all adds if there were any deletes. We definitely don't want that. Also, cleanup of cron scripts in the same commit because I'm lazy Signed-off-by: Aaron Griffin --- cron-jobs/genpkglist | 86 ++++++++++++---------------------------------------- 1 file changed, 20 insertions(+), 66 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/genpkglist b/cron-jobs/genpkglist index 31b7c78..0b5d76f 100755 --- a/cron-jobs/genpkglist +++ b/cron-jobs/genpkglist @@ -1,74 +1,28 @@ #!/bin/bash -# -# genpkglist -# -# Generates a text package database for use with the setup script -# (also used to check for missing packages in the download directory) -# -# Get the package name from the filename -# hackish, but should work for now -getpkgname() { - local tmp +if [ $# -ne 2 ]; then + echo "usage: $(basename $0) " + exit 1 +fi - tmp=${1##*/} - tmp=${tmp%.pkg.tar.gz} - tmp=${tmp%-i686} - tmp=${tmp%-x86_64} - echo ${tmp%-*-*} -} +reponame=$1 +arch=$2 -pkgfile="$(pwd)/packages.txt" -ftppath=$1 -repotag=$2 +##### Arch specific stuff. TODO make this configurable ##### +ftppath="/home/ftp/$reponame/os/$arch/" +############################################################ -rm -f $pkgfile -for package in *; do - if [ -d "$package/$repotag/" ]; then - cd "$package/$repotag/" - if [ -f PKGBUILD ]; then - . PKGBUILD - if [ -f $ftppath/$pkgname-$pkgver-$pkgrel.pkg.tar.gz ]; then - echo "$pkgname-$pkgver-$pkgrel.pkg.tar.gz" >>$pkgfile - elif [ -f $ftppath/$pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz ]; then - echo "$pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz" >>$pkgfile - else - echo "notice: Missing $pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz in ftp site" >&2 - fi - fi - cd .. - fi -done +if [ ! -d "$ftppath" ]; then + echo "FTP path '$ftppath' does not exist" + exit 1 +fi -cd $ftppath -unset DUPES DUPEFILES last -for pkg in *.pkg.tar.gz; do - pkgname=$(getpkgname $pkg) - if [ "$last" = "$pkgname" ]; then - DUPES="$DUPES $pkgname" - DUPEFILES="$DUPEFILES $pkg" - fi - last=$pkgname -done +if [ ! -f /etc/makepkg.conf ]; then + echo "/etc/makepkg.conf not found! Aborting" + exit 1 +fi -showdupes() { - done= - for i in *.pkg.tar.gz; do - pkgname=$(getpkgname $i) - if [ "$pkgname" = "$1" ]; then - ls -l $i | awk '{print $6" "$7" "$8" "$9}' - done=1 - else - [ "$done" = "1" ] && return - fi - done -} +. /etc/makepkg.conf -if [ "$DUPES" ]; then - DUPES=$((for d in $DUPES; do echo $d; done) | sort -u) - echo "Possible Dupes for $ftppath (remove old versions)" - echo "Date Filename" - for dupe in $DUPES; do - showdupes $dupe - done -fi +cd "$ftppath" +ls -1 *$PKGEXT -- cgit v1.2.3-2-g168b From c544d99f907014a5f40356d79b9fb518b559372d Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 15 May 2008 12:05:36 -0400 Subject: More cron job cleanup and fixes Signed-off-by: Aaron Griffin --- cron-jobs/cleanup2.sh | 46 ------------------- cron-jobs/createFileLists | 79 +++++++++++++++++++++++++++++++++ cron-jobs/ftpdir-cleanup | 111 +++++++++++++++++++++++++++++++++------------- 3 files changed, 159 insertions(+), 77 deletions(-) delete mode 100755 cron-jobs/cleanup2.sh create mode 100644 cron-jobs/createFileLists (limited to 'cron-jobs') diff --git a/cron-jobs/cleanup2.sh b/cron-jobs/cleanup2.sh deleted file mode 100755 index 4a79654..0000000 --- a/cron-jobs/cleanup2.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -usage() { - echo "Usage: $0 repo architecture" -} - -getpkgname() { - local tmp - - tmp=${1##*/} - tmp=${tmp%.pkg.tar.gz} - tmp=${tmp%-i686} - tmp=${tmp%-x86_64} - echo ${tmp%-*-*} -} - -FTPBASEDIR="/home/ftp" -FTPDIR=${FTPBASEDIR}/${1}/os/${2} -DBFILE=${FTPDIR}/${1}.db.tar.gz -OBSOLETEFILES="" - -if [ $# -lt 2 -o ! -f ${DBFILE} ]; then - usage - exit 1 -fi - -TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1 -cd ${TMPDIR} -tar xzf ${DBFILE} - -cd ${FTPDIR} -for pkgfile in *.pkg.tar.gz; do - pkgname="$(getpkgname ${pkgfile})" - for p in ${FTPDIR}/${pkgname}-*; do - if [ "$(getpkgname $(basename ${p}))" = "${pkgname}" ]; then - continue 2 - fi - done - OBSOLETEFILES="${OBSOLETEFILES} ${pkgfile}" -done - -cd - >/dev/null -rm -rf ${TMPDIR} - -echo -ne "DIRECTORY:\n${FTPDIR}\n\n" -echo -ne "OBSOLETEFILES:\n${OBSOLETEFILES}\n\n" diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists new file mode 100644 index 0000000..02af296 --- /dev/null +++ b/cron-jobs/createFileLists @@ -0,0 +1,79 @@ +#!/bin/bash + +reposdir=/home/ftp/ +targetdir=/home/pierre/public_html/test-repo/ +repos="core extra unstable testing community" +arches="i686 x86_64" + +if [ -f "/tmp/createFileList.lock" ]; then + echo "Error: createFileList allready in progress." + exit 1 +fi + +touch "/tmp/createFileList.lock" || exit 1 +TMPDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1 +CACHEDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1 + +getpkgname() { + local tmp + + tmp=${1##*/} + tmp=${tmp%.pkg.tar.gz} + tmp=${tmp%-i686} + tmp=${tmp%-x86_64} + tmp=${tmp%-any} + echo $tmp +} + +cd $reposdir +for repo in $repos; do + REPO_DB_FILE=${repo}.files.tar.gz + for arch in $arches; do + repodir=${repo}/os/${arch}/ + cached="no" + + # extract old file archive + if [ -f ${targetdir}${repodir}${REPO_DB_FILE} ]; then + mkdir -p ${CACHEDIR}/${repodir} + bsdtar -xf ${targetdir}${repodir}${REPO_DB_FILE} -C ${CACHEDIR}/${repodir} + cached="yes" + fi + + # create file lists + for pkg in $repodir*.pkg.tar.gz; do + basename=$(basename $pkg) + pkgname=$(getpkgname $basename) + tmppkgdir=${TMPDIR}/${repodir}${pkgname}/ + mkdir -p $tmppkgdir + if [ -f "${CACHEDIR}/${repodir}${pkgname}/files" ]; then +# echo "cache: $pkgname" + mv ${CACHEDIR}/${repodir}${pkgname}/files ${tmppkgdir}files + else +# echo "$repo/$arch: $pkgname" + echo '%FILES%' > ${tmppkgdir}files + bsdtar --exclude=.* -tf $pkg >> ${tmppkgdir}files + cached="no" + fi + done + + # create new file archive + if [ "$cached" == "no" ]; then + # at least one package has changed, so let's rebuild the archive +# echo "creating ${REPO_DB_FILE}/${arch}" + pkgdir=${targetdir}${repodir} + mkdir -p $pkgdir + cd ${TMPDIR}/${repodir} + [ -f "${pkgdir}${REPO_DB_FILE}.old" ] && rm "${pkgdir}${REPO_DB_FILE}.old" + [ -f "${pkgdir}${REPO_DB_FILE}" ] && mv "${pkgdir}${REPO_DB_FILE}" "${pkgdir}${REPO_DB_FILE}.old" + bsdtar --exclude=*.tar.gz -czf ${pkgdir}${REPO_DB_FILE} * + fi + + cd $reposdir + done +done + +cd - >/dev/null +rm -rf $TMPDIR || exit 1 +rm -rf $CACHEDIR || exit 1 +rm -f "/tmp/createFileList.lock" || exit 1 +# echo 'done' diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 18e26f2..83c7cf1 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -1,45 +1,64 @@ #!/bin/bash -usage() { - echo "Usage: $0 repo architecture" -} +if [ $# -ne 2 ]; then + echo "usage: $(basename $0) " + exit 1 +fi + +reponame=$1 +arch=$2 + +##### Arch specific stuff. TODO make this configurable ##### +ftppath="/home/ftp/$reponame/os/$arch/" +############################################################ + +if [ ! -d "$ftppath" ]; then + echo "FTP path '$ftppath' does not exist" + exit 1 +fi + +if [ ! -f /etc/makepkg.conf ]; then + echo "/etc/makepkg.conf not found! Aborting" + exit 1 +fi + +. /etc/makepkg.conf getpkgname() { - local tmp + local tmp - tmp=${1##*/} - tmp=${tmp%.pkg.tar.gz} - tmp=${tmp%-i686} - tmp=${tmp%-x86_64} - echo ${tmp%-*-*} + tmp=${1##*/} + tmp=${tmp%$PKGEXT} + tmp=${tmp%-$arch} + echo ${tmp%-*-*} } -reponame=$1 -arch=$2 +getpkgname_ver() { + local tmp + + tmp=${1##*/} + tmp=${tmp%$PKGEXT} + echo ${tmp%-$arch} +} -FTPBASEDIR="/home/ftp" -FTPDIR=${FTPBASEDIR}/$reponame/os/$arch -DBFILE=${FTPDIR}/$reponame.db.tar.gz MISSINGFILES="" DELETEFILES="" - -if [ $# -lt 2 -o ! -f ${DBFILE} ]; then - usage - exit 1 -fi +EXTRAFILES="" TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1 -cd ${TMPDIR} -tar xzf ${DBFILE} +cd "${TMPDIR}" +/bin/tar xzf "$ftppath/$reponame.db.tar.$DB_COMPRESSION" + for pkg in *; do - filename=$(grep -A1 '^%FILENAME%$' ${pkg}/desc | tail -n1) + filename=$(grep -A1 '^%FILENAME%$' "${pkg}/desc" | tail -n1) [ -z "${filename}" ] && filename="${pkg}.pkg.tar.gz" - if [ ! -f ${FTPDIR}/${filename} ]; then + + if [ ! -f "${ftppath}/${filename}" ]; then MISSINGFILES="${MISSINGFILES} ${filename}" else pkgname="$(getpkgname ${filename})" - for otherfile in ${FTPDIR}/${pkgname}-*; do + for otherfile in ${ftppath}/${pkgname}-*; do otherfile="$(basename ${otherfile})" if [ "${otherfile}" != "${filename}" -a "${pkgname}" = "$(getpkgname ${otherfile})" ]; then DELETEFILES="${DELETEFILES} ${otherfile}" @@ -48,13 +67,43 @@ for pkg in *; do fi done -cd - >/dev/null +cd "$ftppath" +for pkg in *$PKGEXT; do + pkgname="$(getpkgname_ver ${filename})" + if [ ! -d "$TMPDIR/$pkgname" ]; then + EXTRAFILES="$EXTRAFILES $pkg" + fi +done + + + +cd "$ftppath" rm -rf ${TMPDIR} -echo -ne "DIRECTORY:\n${FTPDIR}\n\n" -echo -ne "DELETEFILES:\n${DELETEFILES}\n\n" -echo -ne "MISSINGFILES:\n${MISSINGFILES}\n\n" +if [ -n "${DELETEFILES}" ]; then + #rm -f ${DELETEFILES} + #mv ${DELETEFILES} /home/package-cleanup/ + echo "" +fi -cd ${FTPDIR} -[ -n "${DELETEFILES}" ] && mv ${DELETEFILES} /home/package-cleanup/ -cd - >/dev/null +echo "Scan complete for $reponame ($arch) at ${ftppath}" +if [ -n "$DELETEFILES" ]; then + echo " The following files have been moved to package-cleanup:" + for f in $DELETEFILES; do + echo " $f" + done +fi +echo "" +if [ -n "$MISSINGFILES" ]; then + echo " The following files are missing in the repo:" + for f in $MISSINGFILES; do + echo " $f" + done +fi +echo "" +if [ -n "$EXTRAFILES" ]; then + echo " The following files are in the repo but not the db:" + for f in $EXTRAFILES; do + echo " $f" + done +fi -- cgit v1.2.3-2-g168b From d2ce9d0893421cb35c62edad84bbadd1bf917d9e Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 20 May 2008 23:19:02 -0400 Subject: Added a simple sendmail-ish script to send to the ML Signed-off-by: Aaron Griffin --- cron-jobs/devlist-mailer | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100755 cron-jobs/devlist-mailer (limited to 'cron-jobs') diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer new file mode 100755 index 0000000..b9a27e3 --- /dev/null +++ b/cron-jobs/devlist-mailer @@ -0,0 +1,16 @@ +#!/bin/bash +#Dummy helper to send email to arch-dev +# It does nothing if no output + +#LIST="arch-dev-public@archlinux.org" +LIST="aaronmgriffin@gmail.com" +FROM="cron@archlinux.org" + +stdin="$(cat)" +#echo used to strip whitespace for checking for actual data +if [ -n "$(echo $stdin)" ]; then + echo "Subject: Repository Maintenance $(date +"%d-%m-%Y %H:%M") + From: cron@archlinux.org + + $stdin" | /usr/sbin/sendmail -F$FROM "$LIST" +fi -- cgit v1.2.3-2-g168b From b0792658d9d6fd11c63a737037bd95ef5ccc50f2 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 21 May 2008 14:48:12 -0400 Subject: Switch to output filelists in official repos Signed-off-by: Aaron Griffin --- cron-jobs/createFileLists | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) mode change 100644 => 100755 cron-jobs/createFileLists (limited to 'cron-jobs') diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists old mode 100644 new mode 100755 index 02af296..d6cbd18 --- a/cron-jobs/createFileLists +++ b/cron-jobs/createFileLists @@ -1,7 +1,8 @@ #!/bin/bash reposdir=/home/ftp/ -targetdir=/home/pierre/public_html/test-repo/ +#targetdir=/home/pierre/public_html/test-repo/ +targetdir=$reposdir repos="core extra unstable testing community" arches="i686 x86_64" -- cgit v1.2.3-2-g168b From 8685aaea284147536394969805c9b29c7da71188 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 21 May 2008 14:50:52 -0400 Subject: Copied 'ftpmaint' from root as adjust-permissions This script makes sure we all have write access to the repos Signed-off-by: Aaron Griffin --- cron-jobs/adjust-permissions | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 cron-jobs/adjust-permissions (limited to 'cron-jobs') diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions new file mode 100644 index 0000000..7f7e4c8 --- /dev/null +++ b/cron-jobs/adjust-permissions @@ -0,0 +1,24 @@ +#!/bin/sh + +if [ -f /tmp/.ftpmaint.lck ]; then + exit 0 +fi + +/bin/touch /tmp/.ftpmaint.lck + +cd /home/ftp +/bin/chown -R ftp:ftp-arch core/os/i686 +/bin/chown -R ftp:ftp-arch core/os/x86_64 +/bin/chown -R ftp:ftp-extra {extra,unstable,testing}/os/i686 +/bin/chown -R ftp:ftp-extra {extra,unstable,testing}/os/x86_64 +/bin/chown -R ftp:aur community + +/bin/chmod -R g+w /home/aur/unsupported{,-temp} + +for d in core extra unstable testing community; do + /bin/chmod -R g+w $d/os/i686 + /bin/chmod -R g+w $d/os/x86_64 +done +/bin/chmod 555 /home/ftp + +rm -f /tmp/.ftpmaint.lck -- cgit v1.2.3-2-g168b From 2343985fcc46852f5b4de82c70824fa1dd702bbf Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 21 May 2008 15:33:29 -0400 Subject: Switch EXTRAFILES test to be more accurate New test suggested by Thomas via cleanup2.sh script Signed-off-by: Aaron Griffin --- cron-jobs/ftpdir-cleanup | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 83c7cf1..576dd6e 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -69,41 +69,46 @@ done cd "$ftppath" for pkg in *$PKGEXT; do - pkgname="$(getpkgname_ver ${filename})" - if [ ! -d "$TMPDIR/$pkgname" ]; then - EXTRAFILES="$EXTRAFILES $pkg" + pkgname="$(getpkgname $pkg)" + if [ "$pkg" != "$TMPDIR/$pkgname-*" -a "$(getpkgname $(basename $pkg))" = "$pkgname" ]; then + continue 2 fi + EXTRAFILES="$EXTRAFILE $pkg" done - - cd "$ftppath" rm -rf ${TMPDIR} -if [ -n "${DELETEFILES}" ]; then - #rm -f ${DELETEFILES} - #mv ${DELETEFILES} /home/package-cleanup/ - echo "" -fi - echo "Scan complete for $reponame ($arch) at ${ftppath}" if [ -n "$DELETEFILES" ]; then - echo " The following files have been moved to package-cleanup:" + echo " The following files are out of date" + echo " They will be moved to /home/package-cleanup" for f in $DELETEFILES; do echo " $f" done fi echo "" if [ -n "$MISSINGFILES" ]; then - echo " The following files are missing in the repo:" + echo " The following files are missing in the repo" for f in $MISSINGFILES; do echo " $f" done fi echo "" if [ -n "$EXTRAFILES" ]; then - echo " The following files are in the repo but not the db:" + echo " The following files are in the repo but not the db" + echo " They will be moved to /home/package-cleanup" for f in $EXTRAFILES; do echo " $f" done fi + +if [ -n "${DELETEFILES}" ]; then + #mv ${DELETEFILES} /home/package-cleanup/ + echo "" +fi + +if [ -n "${EXTRAFILES}" ]; then + #mv ${EXTRAFILES} /home/package-cleanup/ + echo "" +fi -- cgit v1.2.3-2-g168b From 603d737cd871d4195dd241c345a133989ce59d28 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 21 May 2008 15:34:15 -0400 Subject: Adjust permissions of the adjust-permissions script HAH! Signed-off-by: Aaron Griffin --- cron-jobs/adjust-permissions | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 cron-jobs/adjust-permissions (limited to 'cron-jobs') diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions old mode 100644 new mode 100755 -- cgit v1.2.3-2-g168b From d1f0d5732167d142af48b48a84bbbbd2adb7593a Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 22 May 2008 17:18:27 -0400 Subject: Git tells me this file was modifed I don't know what changed, but I am committing to check! Signed-off-by: Aaron Griffin --- cron-jobs/ftpdir-cleanup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 576dd6e..d6dce11 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -104,11 +104,11 @@ if [ -n "$EXTRAFILES" ]; then fi if [ -n "${DELETEFILES}" ]; then - #mv ${DELETEFILES} /home/package-cleanup/ + mv ${DELETEFILES} /home/package-cleanup/ echo "" fi if [ -n "${EXTRAFILES}" ]; then - #mv ${EXTRAFILES} /home/package-cleanup/ + mv ${EXTRAFILES} /home/package-cleanup/ echo "" fi -- cgit v1.2.3-2-g168b From 6f67fcda30b1589f1716fb64191c752864e9fab7 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 22 May 2008 17:20:32 -0400 Subject: crontab file for repo-based cron jobs Signed-off-by: Aaron Griffin --- cron-jobs/repo-maint.crontab | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 cron-jobs/repo-maint.crontab (limited to 'cron-jobs') diff --git a/cron-jobs/repo-maint.crontab b/cron-jobs/repo-maint.crontab new file mode 100644 index 0000000..dadd1fb --- /dev/null +++ b/cron-jobs/repo-maint.crontab @@ -0,0 +1,12 @@ +MAILTO="root" +#################################################################### +#minute (0-59), # +#| hour (0-23), # +#| | day of the month (1-31), # +#| | | month of the year (1-12), # +#| | | | day of the week (0-6 with 0=Sunday)# +#| | | | | commands # +#################################################################### +*/5 * * * * /arch/cron-jobs/adjust-permissions +00 21 * * * /arch/cron-jobs/createFileList +00 */3 * * * /arch/cron-jobs/ftpdir-cleanup | /arch/cron-jobs/devlist-mailer -- cgit v1.2.3-2-g168b From 1c9a21bdf378ddd3572b3b8f405d99b3063521c9 Mon Sep 17 00:00:00 2001 From: Scott Horowitz Date: Tue, 27 May 2008 14:34:00 -0400 Subject: Add the check_archlinux.py repo checker This is to be used for integrity checked, generated perhaps once a day Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux.py | 724 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 724 insertions(+) create mode 100755 cron-jobs/check_archlinux.py (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux.py b/cron-jobs/check_archlinux.py new file mode 100755 index 0000000..8e39c81 --- /dev/null +++ b/cron-jobs/check_archlinux.py @@ -0,0 +1,724 @@ +#!/usr/bin/env python +# +# check_archlinux.py +# +# Written by Scott Horowitz +# Graphical dependency tree by Cesar G. Miguel +# +# This script currently checks for a number of issues in your ABS tree: +# 1. Directories with missing PKGBUILDS +# 2. Duplicate PKGBUILDs +# 3. Missing (make-)dependencies, taking into account 'provides' +# 4. Provisioned dependencies +# 5. Circular dependencies +# 6. Valid arch's in PKGBUILDS +# 7. Missing packages in Arch repos +# 8. PKGBUILD names that don't match their directory +# 9. Hierarchy of repos (e.g., that a core package doesn't depend on +# a non-core package) +# It can also, optionally, generate a graphical representation of the +# dependency tree. +# +# Todo: +# 1. Accommodate testing repo? + +abs_conf_dir = "/etc/abs" +valid_archs = ['i686', 'x86_64'] +cvs_tags = {'i686': 'CURRENT', 'x86_64': 'CURRENT-64'} +include_paths = ['core', 'extra', 'community', 'unstable'] + +base_server = "ftp.archlinux.org" # Must be ftp site +# Ensure that core repo only depends on core, and the extra repo only +# depends on core and extra, etc. +repo_hierarchy = {'core': ('core'), \ +'extra': ('core', 'extra'), \ +'unstable': ('core', 'extra', 'unstable'), \ +'community': ('core', 'extra', 'unstable', 'community')} + +pkgname_str = "pkgname=" +dep_str = "depends=(" +makedep_str = "makedepends=(" +provides_str = "provides=(" +arch_str = "arch=(" +pkgver_str = "pkgver=" +pkgrel_str = "pkgrel=" +build_str = "build()" +source_str = "source=" +url_str = "url=" +mathlist = (">=", "<=", "=", ">", "<") +sup_tag_str = "*default tag=" + +import os, sys, getopt, tempfile +from ftplib import FTP +try: + import pydot + HAS_PYDOT = True +except: + HAS_PYDOT = False + +def print_heading(heading): + print "" + print "=" * (len(heading) + 4) + print "= " + heading + " =" + print "=" * (len(heading) + 4) + +def print_subheading(subheading): + print "" + print subheading + print "-" * (len(subheading) + 2) + +def rmgeneric(path, __func__): + try: + __func__(path) + except OSError, (errno, strerror): + pass + +def removeall(path): + if not os.path.isdir(path): + return + + files=os.listdir(path) + + for x in files: + fullpath=os.path.join(path, x) + if os.path.isfile(fullpath): + f=os.remove + rmgeneric(fullpath, f) + elif os.path.isdir(fullpath): + removeall(fullpath) + f=os.rmdir + rmgeneric(fullpath, f) + +def update_var(line, user_vars, pkgpath): + if line.count("$") > 0: + export_line = "" + for var in user_vars: + if line.count(var[0]) > 0: + export_line = export_line + var[0] + "=" + var[1] + " && " + if line.startswith("(") and line.endswith(")"): + line = line[1:-1] + export_line = export_line + "echo \"" + line + "\"" + line = os.popen(export_line).read().replace("\n", "") + return line + +def split_dep_prov_symbol(dep): + # Splits 'foo>=1.2.3' into ('foo', '1.2.3', '>=') + prov = "" + symbol = "" + for char in mathlist: + pos = dep.find(char) + if pos > -1: + prov = dep[pos:].replace(char, "") + dep = dep[:pos] + symbol = char + break + return (dep, prov, symbol) + +def repo_from_path(path): + return path.split("/")[-4] + +def create_supfile(sourcefile, destfile, newtag): + o = open(sourcefile, 'r') + info = o.read() + o.close() + lines = info.split("\n") + + o = open(destfile, 'w') + for line in lines: + line = line.strip() + if line[:len(sup_tag_str)] == sup_tag_str: + line = sup_tag_str + newtag + o.write(line + "\n") + o.close() + +def get_deps_provides_etc(pkgpath): + # Parse PKGBUILD for name, depends, makedepends, provides, arch's, and version + o = open(pkgpath, 'r') + info = o.read() + o.close() + lines = info.split("\n") + + deps = [] + provides = [] + archs = [] + makedeps = [] + array = [] + user_vars = [] + continue_line = False + for line in lines: + line = line.strip() + if line.find("#") > -1: + line = line[:line.find("#")].strip() + if not continue_line: + deps_line = False + provides_line = False + arch_line = False + makedeps_line = False + if line[:len(dep_str)] == dep_str: + line = line.replace(dep_str,"") + deps_line = True + elif line[:len(makedep_str)] == makedep_str: + line = line.replace(makedep_str,"") + makedeps_line = True + elif line[:len(provides_str)] == provides_str: + line = line.replace(provides_str,"") + provides_line = True + elif line[:len(arch_str)] == arch_str: + line = line.replace(arch_str, "") + arch_line = True + elif line[:len(pkgname_str)] == pkgname_str: + pkgname = line.replace(pkgname_str, "") + if pkgname.startswith("\"") and pkgname.endswith("\""): + pkgname = pkgname[1:-1] + pkgname = update_var(pkgname, user_vars, pkgpath) + user_vars.append([pkgname_str, pkgname]) + line = "" + elif line[:len(pkgver_str)] == pkgver_str: + pkgver = line.replace(pkgver_str, "") + if pkgver.startswith("\"") and pkgver.endswith("\""): + pkgver = pkgver[1:-1] + pkgver = update_var(pkgver, user_vars, pkgpath) + user_vars.append([pkgver_str[:-1], pkgver]) + line = "" + elif line[:len(pkgrel_str)] == pkgrel_str: + pkgrel = line.replace(pkgrel_str, "") + if pkgrel.startswith("\"") and pkgrel.endswith("\""): + pkgrel = pkgrel[1:-1] + pkgrel = update_var(pkgrel, user_vars, pkgpath) + user_vars.append([pkgrel_str[:-1], pkgrel]) + line = "" + elif line[:len(build_str)] == build_str: + break + elif not continue_line: + if line.count("=") == 1 and line.count(" ") == 0 and line[:1] != "#" and \ + line[:len(source_str)] != source_str and line[:len(url_str)] != url_str: + split = line.split("=") + for item in range(len(split)): + split[item] = update_var(split[item], user_vars, pkgpath) + user_vars.append(split) + line = "" + if len(line) > 0: + pos = line.find(")") + if pos > -1: + # strip everything from closing paranthesis on + # since some PKGBUILDS have comments after the + # depends array + line = line[:pos] + line = line.split(' ') + for i in range(len(line)): + line[i] = line[i].replace("'","").replace('"','') + line[i] = update_var(line[i], user_vars, pkgpath) + if len(line[i]) > 0: + if deps_line: + deps.append(line[i]) + array=deps + elif provides_line: + provides.append(line[i]) + array=provides + elif arch_line: + archs.append(line[i]) + array=archs + elif makedeps_line: + makedeps.append(line[i]) + array=makedeps + if array and (array[-1] == "\\" or array[-1][-1] == "\\"): + # continue reading deps/provides on next line + if array[-1] == "\\": + array.pop(-1) + else: + array[-1] = array[-1].replace("\\", "") + continue_line = True + else: + continue_line = False + version = pkgver + "-" + pkgrel + return (pkgname, deps, makedeps, provides, archs, version) + +def get_pkgbuilds_in_dir(rootdir): + # Recursively populates pkgbuild_deps, pkgbuild_paths, etc. + # dicts with info from each PKGBUILD found in rootdir: + if rootdir != absroot: + if rootdir.count("/") == (absroot.count("/")+1) and rootdir not in curr_include_paths: + return + pkgfound = False + for f in os.listdir(rootdir): + fpath = rootdir + "/" + f + if os.path.isdir(fpath): + get_pkgbuilds_in_dir(fpath) + elif f == 'PKGBUILD': + pkgfound = True + name = rootdir.split("/")[-1] + if name in pkgbuild_deps: + dups.append(fpath.replace(absroot, "") + " vs. " + pkgbuild_paths[name].replace(absroot, "")) + else: + (pkgname, deps, makedeps, provides, archs, version) = get_deps_provides_etc(fpath) + pkgbuild_deps[pkgname] = deps + pkgbuild_makedeps[pkgname] = makedeps + pkgbuild_paths[pkgname] = fpath + pkgbuild_archs[pkgname] = archs + pkgbuild_versions[pkgname] = version + # We'll store the provides "backwards" compared to + # the other dicts. This will make searching for + # provides easier by being able to look up the + # provide name itself and find what provides it + for provide in provides: + pkgbuild_provides[provide] = pkgname + if pkgname != name: + mismatches.append(pkgname + " vs. " + fpath.replace(absroot, "")) + if not pkgfound and rootdir.replace(absroot, "").count("/") == 3: + misses.append(rootdir.replace(absroot, "") + "/PKGBUILD") + +def verify_depends_makedepends(verify_makedeps=False): + # Make sure all the deps we parsed are actually packages; also + # ensure we meet dep provisions. + if verify_makedeps: + array = pkgbuild_makedeps + else: + array = pkgbuild_deps + for pkgname in array: + deps = array[pkgname] + pkg_repo = repo_from_path(pkgbuild_paths[pkgname]) + deps_to_pop = [] + for i in range(len(deps)): + dep = deps[i] + (dep, prov, char) = split_dep_prov_symbol(dep) + try: + x = pkgbuild_deps[dep] + # Check that prov is met too: + if len(prov) > 0: + compare_str = "vercmp " + pkgbuild_versions[dep] + " " + prov + error = False + if char == "<=": + if int(os.popen(compare_str).read().replace("\n", "")) > 0: + error = True + elif char == ">=": + if int(os.popen(compare_str).read().replace("\n", "")) < 0: + error = True + elif char == "=": + if int(os.popen(compare_str).read().replace("\n", "")) != 0: + error = True + elif char == ">": + if int(os.popen(compare_str).read().replace("\n", "")) <= 0: + error = True + elif char == "<": + if int(os.popen(compare_str).read().replace("\n", "")) >= 0: + error = True + if error: + if verify_makedeps: + unmet_makedep_provs.append(pkgname + " --> '" + dep + char + prov + "'") + else: + unmet_dep_provs.append(pkgname + " --> '" + dep + char + prov + "'") + # Check repos fit hierarchy scheme: + dep_repo = repo_from_path(pkgbuild_paths[dep]) + try: + valid_repos = repo_hierarchy[pkg_repo] + # Make sure dep repo is one of the valid repos: + if dep_repo not in valid_repos: + if verify_makedeps: + makedep_hierarchy.append(pkg_repo + "/" + pkgname + " depends on " + dep_repo + "/" + dep) + else: + dep_hierarchy.append(pkg_repo + "/" + pkgname + " depends on " + dep_repo + "/" + dep) + except: + pass + except: + # Check if a package provides this dep: + try: + x = pkgbuild_provides[dep] + except: + if verify_makedeps: + missing_makedeps.append(pkgname + " --> '" + dep + "'") + else: + missing_deps.append(pkgname + " --> '" + dep + "'") + deps_to_pop.append(i) + # Pop deps not found from end to beginning: + while len(deps_to_pop) > 0: + deps.pop(deps_to_pop[-1]) + deps_to_pop.pop(-1) + +def verify_archs(): + for pkgname in pkgbuild_archs: + newarch = [] + archs = pkgbuild_archs[pkgname] + for arch in archs: + if arch not in valid_archs: + invalid_archs.append(pkgname + " --> " + arch) + else: + newarch.append(arch) + if len(newarch) > 0: + pkgbuild_archs[pkgname] = newarch + +def verify_packages(tree_arch): + # Make sure packages exist in Arch repo(s): + ftp = FTP(base_server) + ftp.login() + prev_wd = "" + # Find all repos/archs; marching through them in order will greatly speed + # up searching for ftp files by minimizing the number of .nlst's that we + # have to do. + repos = [] + for pkgname in pkgbuild_paths: + pkgrepo = repo_from_path(pkgbuild_paths[pkgname]) + if not pkgrepo in repos: + repos.append(pkgrepo) + archs = [] + for pkgname in pkgbuild_archs: + pkgarchs = pkgbuild_archs[pkgname] + for arch in pkgarchs: + if not arch in archs: + archs.append(arch) + for r in repos: + for pkgname in pkgbuild_archs: + repo = repo_from_path(pkgbuild_paths[pkgname]) + if repo == r: + archs = pkgbuild_archs[pkgname] + pkgver_rel = pkgbuild_versions[pkgname] + for arch in archs: + if arch == tree_arch: + # Check for file: + wd = repo + "/os/" + arch + if wd != prev_wd: + ftpfiles = ftp.nlst(wd) + prev_wd = wd + fname_new = wd + "/" + pkgname + "-" + pkgver_rel + "-" + arch + ".pkg.tar.gz" + fname_old = wd + "/" + pkgname + "-" + pkgver_rel + ".pkg.tar.gz" + if fname_old not in ftpfiles and fname_new not in ftpfiles: + missing_pkgs.append(pkgname + "-" + pkgver_rel + " in " + wd) + ftp.quit() + +def subset_superset_path(currpath, currpathnum, paths): + # If a pair of subset/superset paths are found, + # pop the superset one to show the more minimal + # case. + # + # e.g. foo > bar > baz > foo (superset) + # foo > bar > foo (subset) + # --> pop the superset + # + currdeps = currpath.split(">") + currdeps = list(set(currdeps)) + currdeps.sort() + pathnum = 0 + for path in paths: + if pathnum != currpathnum: + deps = path.split(">") + deps = list(set(deps)) + deps.sort() + if len(currdeps) < len(path): + subset = True + for d in currdeps: + if not d in path: + subset = False + break + if subset: + circular_deps.pop(pathnum) + if pathnum <= currpathnum: + currpathnum -= 1 + elif len(currdeps) > len(path): + superset = True + for d in path: + if not d in currdeps: + superset = False + break + if superset: + circular_deps.pop(currpathnum) + currpathnum -= 1 + pathnum += 1 + return True + +def unique_path(currpath, paths): + # Returns false if an equivalent path exists in + # paths. + # + # e.g. foo > bar > foo + # bar > foo > bar + # + currdeps = currpath.split(">") + currdeps = list(set(currdeps)) + currdeps.sort() + for path in paths: + deps = path.split(">") + deps = list(set(deps)) + deps.sort() + if currdeps == deps: + return False + return True + +def update_paths(paths, dep, dep2): + # Update paths by appending new paths with dep2 + # based on all the different ways we could get + # to dep2. Returns True if a path was updated. + new_path = False + for i in range(len(paths)): + array = paths[i].split(">") + newpath = paths[i] + ">" + dep2 + if array[-1] == dep and not dep2 in array and unique_path(newpath, paths): + paths.append(newpath) + new_path = True + return new_path + +def check_name_in_recursive_deps(pkgname): + # Retrieve all recursive dependencies from a package and + # determines if pkgname is found in one of its deps. + recursive_deps = [] + for dep in pkgbuild_deps[pkgname]: + dep = split_dep_prov_symbol(dep)[0] # Strip any provision + recursive_deps.append(dep) + paths = [] + for dep in recursive_deps: + dep = split_dep_prov_symbol(dep)[0] # Strip any provision + paths.append(dep) + searching = True + while searching: + searching = False + for dep in recursive_deps: + for dep2 in pkgbuild_deps[dep]: + dep2 = split_dep_prov_symbol(dep2)[0] # Strip any provision + # This is a HUGE time-saver. Instead of generating every single + # possible path that can yield a circular dep, we'll reduce + # the number greatly by throwing out paths like such: + # + # If we have a path: foo>bar>baz>blah>poop>foo + # We will throw out any: + # foo>...>bar>baz>blah>poop>foo + # foo>...>baz>blah>poop>foo + # foo>...>blah>poop>foo + # and so on. Otherwise we will find hundreds or even thousands + # of possible paths that all essentially represent the same + # circular dep. + # + # However, we will always let pkgname through in order to make + # sure we can find multiple circular deps for a given pkg. + if dep2 not in recursive_deps or dep2 == pkgname: + updated = update_paths(paths, dep, dep2) + if dep2 not in recursive_deps: + recursive_deps.append(dep2) + if updated: + searching = True + # Done searching, store circular deps: + for path in paths: + if path.split(">")[-1] == pkgname: + if unique_path(pkgname + ">" + path, circular_deps): + circular_deps.append(pkgname + ">" + path) + # Reduce any subset/superset path pairs: + pathnum = 0 + for path in circular_deps: + subset_superset_path(path, pathnum, circular_deps) + pathnum += 1 + +def circular_deps_check(): + # Check for circular dependencies: + for pkgname in pkgbuild_deps: + check_name_in_recursive_deps(pkgname) + +def visualize_repo(): + output = 'digraph G { \n \ + concentrate = true; \n \ + ordering = out; \n \ + ranksep=5.0; \n \ + node [style=filled,fontsize=8]; \n' + + # draws circular dependencies in red + for path in circular_deps: + output += '\t "'+path[0]+'"' + deps = path.split(">") + for d in deps: + output += ' -> "'+d+'"' + output += ' [color=red]\n' + + for pkg in pkgbuild_deps.keys(): + output += '\t "'+pkg+'" -> { ' + for d in pkgbuild_deps[pkg]: + d = split_dep_prov_symbol(d)[0] # Strip any provision + output += '"'+d+'"; ' + output += '}\n' + + output += '}' + + # Uncomment these lines to get a file dump called + # 'output'. This can be used to manually generate + # an image using, e.g., dot -Tsvg output tree.svg + #dump = open('output', 'w') + #dump.write(output) + #dump.close() + + fname = 'dependency_tree-' + arch + '.svg' + print "Generating " + fname + "..." + g = pydot.graph_from_dot_data(output) + g.write(fname, prog='dot', format='svg') + +def print_result(list, subheading): + if len(list) > 0: + print_subheading(subheading) + for item in list: + print item + +def print_results(): + print_result(misses, "Missing PKGBUILDs") + print_result(mismatches, "Mismatched Pkgnames") + print_result(dups, "Duplicate PKGBUILDs") + print_result(missing_deps, "Missing Dependencies") + print_result(missing_makedeps, "Missing Makedepends") + print_result(unmet_dep_provs, "Unmet Dependency Provisions") + print_result(unmet_makedep_provs, "Unmet Makedepends Provisions") + print_result(dep_hierarchy, "Repo Hierarchy for Dependencies") + print_result(makedep_hierarchy, "Repo Hierarchy for Makedepends") + print_result(invalid_archs, "Invalid Archs") + print_result(circular_deps, "Circular Dependencies") + print_result(missing_pkgs, "Missing Repo Packages") + print_subheading("Summary") + print "Dirs with missing PKGBUILDs: ", len(misses) + print "Duplicate PKGBUILDs: ", len(dups) + print "Missing (make)dependencies: ", len(missing_deps)+len(missing_makedeps) + print "Unmet provisioned (make)dependencies: ", len(unmet_dep_provs)+len(unmet_makedep_provs) + print "Circular dependencies: ", len(circular_deps) + print "Invalid archs: ", len(invalid_archs) + print "Missing packages in repos: ", len(missing_pkgs) + print "Mismatching PKGBUILD names: ", len(mismatches) + print "Repo hierarchy problems: ", len(dep_hierarchy)+len(makedep_hierarchy) + print "" + +def print_usage(): + print "" + print "Usage: check_archlinux [OPTION]" + print "" + print "Options:" + print " --abs-tree= Check specified tree (assumes the abs tree" + print " is i686 unless overridden with --arch)" + print " --arch= Use specified arch (e.g. 'x86_64')" + print " -g Generate graphical dependency tree(s)" + print " -h, --help Show this help and exit" + print "" + print "Examples:" + print "\n Check all arches and do fresh cvs checkouts:" + print " check_archlinux" + print "\n Check x_86_64 only and do fresh cvs checkout:" + print " check_archlinux --arch=x86_64" + print "\n Check existing i686 abs tree:" + print " check_archlinux --abs-tree=/var/abs" + print "\n Check existing x86_64 abs tree and also generate dep tree image:" + print " check_archlinux --abs-tree=/var/abs --arch=x86_64 -g" + print "" + +graphdeptree = False +user_absroot = "" +user_arch = "" +try: + opts, args = getopt.getopt(sys.argv[1:], "g", ["abs-tree=", "arch="]) +except getopt.GetoptError: + print_usage() + sys.exit() +if opts != []: + for o, a in opts: + if o in ("-g"): + graphdeptree = True + if not HAS_PYDOT: + print "You must install pydot to generate a graphical dependency tree. Aborting..." + sys.exit() + elif o in ("--abs-tree"): + user_absroot = a + elif o in ("--arch"): + user_arch = a + if user_arch not in valid_archs: + print "You did not specify a valid arch. Aborting..." + sys.exit() + else: + print_usage() + sys.exit() + if args != []: + for a in args: + if a in ("play", "pause", "stop", "next", "prev", "pp", "info", "status", "repeat", "shuffle"): + self.single_connect_for_passed_arg(a) + else: + self.print_usage() + sys.exit() + +if len(user_absroot) > 0 and len(user_arch) == 0: + user_arch = valid_archs[0] # i686 default.. + +if len(user_absroot) > 0: + print "Warning: Ensure your ABS tree is clean to prevent false positives." + +try: + absroots = [] + fsup = 'supfile.curr' + for arch in valid_archs: + if len(user_arch) == 0 or user_arch == arch: + print_heading(arch + " Integrity Check") + if len(user_absroot) == 0: + # Create temp dir for cvs checkout: + absroot = tempfile.mkdtemp(prefix="abs-" + arch + "-") + absroots.append(absroot) + print "\nChecking out clean abs tree" + for repo in include_paths: + print "==>", repo + # Create (and modify) supfile: + if not os.path.exists(abs_conf_dir + '/supfile.' + repo): + print "Cannot find file " + abs_conf_dir + '/supfile.' + repo + ". Aborting..." + sys.exit() + create_supfile(abs_conf_dir + '/supfile.' + repo, fsup, cvs_tags[arch]) + cmd = 'csup -L 0 -r 0 -g -b ' + absroot + ' -c .sup ' + fsup + os.system(cmd) + else: + absroot = user_absroot + absroots.append(absroot) + curr_include_paths = [] + for repo in include_paths: + curr_include_paths.append(absroot + "/" + repo) + + # Re-init vars for new abs tree: + pkgbuild_deps = {} # pkgname: [dep1, dep2>foo, dep3=bar, ...] + pkgbuild_makedeps = {} # pkgname: [dep1, dep2>foo, dep3=bar, ...] + pkgbuild_provides = {} # provide_name: pkgname + pkgbuild_paths = {} # pkgname: /var/abs/foo/bar/pkgname/PKGBUILD + pkgbuild_archs = {} # pkgname: [i686, x86_64] + pkgbuild_versions = {} # pkname: 1.0.4-2 + # circular_deps is not a dict to accommodate multiple circ deps for a given pkg + circular_deps = [] # pkgname>dep1>dep2>...>pkgname + mismatches = [] + misses = [] + dups = [] + missing_deps = [] + unmet_dep_provs = [] + dep_hierarchy = [] + missing_makedeps = [] + unmet_makedep_provs = [] + makedep_hierarchy = [] + invalid_archs = [] + missing_pkgs = [] + + # Verify stuff for abs tree: + print "\nPerforming integrity checks..." + print "==> parsing pkgbuilds" + get_pkgbuilds_in_dir(absroot) + print "==> checking dependencies" + verify_depends_makedepends() + print "==> checking makedepends" + verify_depends_makedepends(True) + print "==> checking archs" + verify_archs() + print "==> checking for circular dependencies" + circular_deps_check() + print "==> checking repo packages" + verify_packages(arch) + print_results() + if graphdeptree: + visualize_repo() + + print "\nCleaning up temporary abs tree..." + if len(user_absroot) == 0: + # Delete temp abs tree: + if os.path.exists(absroot): + removeall(absroot) + os.rmdir(absroot) +except: + # Cleanup files... + if len(user_absroot) == 0: + print "\nCleaning up temporary abs tree(s)..." + # Delete any temp abs trees: + for path in absroots: + if os.path.exists(path): + removeall(path) + os.rmdir(path) +finally: + # Remove supfile: + if os.path.exists(fsup): + os.remove(fsup) -- cgit v1.2.3-2-g168b From c417aa306de4329322d99f1067f97a35013610b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20B=C3=A4chler?= Date: Thu, 29 May 2008 15:29:56 -0400 Subject: Fix EXTRAFILES in ftpdir-cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Thomas Bächler Signed-off-by: Aaron Griffin --- cron-jobs/ftpdir-cleanup | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index d6dce11..f8c1a8e 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -70,10 +70,12 @@ done cd "$ftppath" for pkg in *$PKGEXT; do pkgname="$(getpkgname $pkg)" - if [ "$pkg" != "$TMPDIR/$pkgname-*" -a "$(getpkgname $(basename $pkg))" = "$pkgname" ]; then - continue 2 - fi - EXTRAFILES="$EXTRAFILE $pkg" + for p in ${TMPDIR}/${pkgname}-*; do + if [ -d "${p}" -a "$(getpkgname $(basename ${p}))" = "${pkgname}" ]; then + continue 2 + fi + done + EXTRAFILES="$EXTRAFILES $pkg" done cd "$ftppath" -- cgit v1.2.3-2-g168b From fd921f8dc355ac51a8be843a4da052474b22700c Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 29 May 2008 15:31:21 -0400 Subject: Move ftpdir-cleanup to misc-scripts Signed-off-by: Aaron Griffin --- cron-jobs/ftpdir-cleanup | 116 ----------------------------------------------- 1 file changed, 116 deletions(-) delete mode 100755 cron-jobs/ftpdir-cleanup (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup deleted file mode 100755 index f8c1a8e..0000000 --- a/cron-jobs/ftpdir-cleanup +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash - -if [ $# -ne 2 ]; then - echo "usage: $(basename $0) " - exit 1 -fi - -reponame=$1 -arch=$2 - -##### Arch specific stuff. TODO make this configurable ##### -ftppath="/home/ftp/$reponame/os/$arch/" -############################################################ - -if [ ! -d "$ftppath" ]; then - echo "FTP path '$ftppath' does not exist" - exit 1 -fi - -if [ ! -f /etc/makepkg.conf ]; then - echo "/etc/makepkg.conf not found! Aborting" - exit 1 -fi - -. /etc/makepkg.conf - -getpkgname() { - local tmp - - tmp=${1##*/} - tmp=${tmp%$PKGEXT} - tmp=${tmp%-$arch} - echo ${tmp%-*-*} -} - -getpkgname_ver() { - local tmp - - tmp=${1##*/} - tmp=${tmp%$PKGEXT} - echo ${tmp%-$arch} -} - -MISSINGFILES="" -DELETEFILES="" -EXTRAFILES="" - -TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1 - -cd "${TMPDIR}" -/bin/tar xzf "$ftppath/$reponame.db.tar.$DB_COMPRESSION" - -for pkg in *; do - filename=$(grep -A1 '^%FILENAME%$' "${pkg}/desc" | tail -n1) - [ -z "${filename}" ] && filename="${pkg}.pkg.tar.gz" - - if [ ! -f "${ftppath}/${filename}" ]; then - MISSINGFILES="${MISSINGFILES} ${filename}" - else - pkgname="$(getpkgname ${filename})" - for otherfile in ${ftppath}/${pkgname}-*; do - otherfile="$(basename ${otherfile})" - if [ "${otherfile}" != "${filename}" -a "${pkgname}" = "$(getpkgname ${otherfile})" ]; then - DELETEFILES="${DELETEFILES} ${otherfile}" - fi - done - fi -done - -cd "$ftppath" -for pkg in *$PKGEXT; do - pkgname="$(getpkgname $pkg)" - for p in ${TMPDIR}/${pkgname}-*; do - if [ -d "${p}" -a "$(getpkgname $(basename ${p}))" = "${pkgname}" ]; then - continue 2 - fi - done - EXTRAFILES="$EXTRAFILES $pkg" -done - -cd "$ftppath" -rm -rf ${TMPDIR} - -echo "Scan complete for $reponame ($arch) at ${ftppath}" -if [ -n "$DELETEFILES" ]; then - echo " The following files are out of date" - echo " They will be moved to /home/package-cleanup" - for f in $DELETEFILES; do - echo " $f" - done -fi -echo "" -if [ -n "$MISSINGFILES" ]; then - echo " The following files are missing in the repo" - for f in $MISSINGFILES; do - echo " $f" - done -fi -echo "" -if [ -n "$EXTRAFILES" ]; then - echo " The following files are in the repo but not the db" - echo " They will be moved to /home/package-cleanup" - for f in $EXTRAFILES; do - echo " $f" - done -fi - -if [ -n "${DELETEFILES}" ]; then - mv ${DELETEFILES} /home/package-cleanup/ - echo "" -fi - -if [ -n "${EXTRAFILES}" ]; then - mv ${EXTRAFILES} /home/package-cleanup/ - echo "" -fi -- cgit v1.2.3-2-g168b From a90b2f5dc26474c291bd15b4bbddc75afea2731f Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 29 May 2008 15:34:49 -0400 Subject: Add a real cron script to cycle over all repos Signed-off-by: Aaron Griffin --- cron-jobs/ftpdir-cleanup | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 cron-jobs/ftpdir-cleanup (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup new file mode 100755 index 0000000..ce91b62 --- /dev/null +++ b/cron-jobs/ftpdir-cleanup @@ -0,0 +1,35 @@ +#!/bin/bash + +#TODO add community +repos="core extra unstable testing" +arches="i686 x86_64" + +LOCKFILE="/tmp/.ftpdircleanup.lock" + +cleanup () { + rm -f "$LOCKFILE" + exit 0 +} + +ctrl_c() { + cleanup +} + +if [ -f "$LOCKFILE" ]; then + owner="$(/usr/bin/stat -c %U $LOCKFILE)" + echo "error: source tarball generation is already in progress (started by $owner)" + exit 1 +fi + +trap cleanup 0 +trap ctrl_c 2 + +/bin/touch "$LOCKFILE" + +for repo in $repos; do + for arch in $arches; do + $(dirname $0)/../misc-scripts/ftpdir-cleanup $repo $arch + done +done + +cleanup -- cgit v1.2.3-2-g168b From 3d287cbad91a35158444b49840ebfc3f86e6e59a Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 18 Jun 2008 18:07:08 -0400 Subject: Correct the devlist mailer script to send real emails Signed-off-by: Aaron Griffin --- cron-jobs/devlist-mailer | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer index b9a27e3..094d60d 100755 --- a/cron-jobs/devlist-mailer +++ b/cron-jobs/devlist-mailer @@ -2,15 +2,16 @@ #Dummy helper to send email to arch-dev # It does nothing if no output -#LIST="arch-dev-public@archlinux.org" -LIST="aaronmgriffin@gmail.com" -FROM="cron@archlinux.org" +LIST="arch-dev-public@archlinux.org" +#LIST="aaronmgriffin@gmail.com" +FROM="repomaint" stdin="$(cat)" #echo used to strip whitespace for checking for actual data if [ -n "$(echo $stdin)" ]; then - echo "Subject: Repository Maintenance $(date +"%d-%m-%Y %H:%M") - From: cron@archlinux.org - $stdin" | /usr/sbin/sendmail -F$FROM "$LIST" +echo "Subject: Repository Maintenance $(date +"%d-%m-%Y %H:%M") + +$stdin" | /usr/sbin/sendmail -F$FROM "$LIST" + fi -- cgit v1.2.3-2-g168b From 5cf896673b4b73d6591f52acbcba5a1dfc803f6c Mon Sep 17 00:00:00 2001 From: Travis Willard Date: Sun, 1 Jun 2008 17:44:09 -0400 Subject: Patch check_archlinux.py to work on our current ABS tree. Patch this on devtools - to run against /home/abs, we'd use: ./check_archlinux.py --abs-tree=/home/abs/rsync/i686 --arch=i686 ./check_archlinux.py --abs-tree=/home/abs/rsync/x86_64 --arch=x86_64 Signed-off-by: Travis Willard Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux.py | 81 ++++++++++++++++---------------------------- 1 file changed, 30 insertions(+), 51 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux.py b/cron-jobs/check_archlinux.py index 8e39c81..055c0a4 100755 --- a/cron-jobs/check_archlinux.py +++ b/cron-jobs/check_archlinux.py @@ -26,6 +26,7 @@ abs_conf_dir = "/etc/abs" valid_archs = ['i686', 'x86_64'] cvs_tags = {'i686': 'CURRENT', 'x86_64': 'CURRENT-64'} include_paths = ['core', 'extra', 'community', 'unstable'] +pkgdir_path_depth = 3 base_server = "ftp.archlinux.org" # Must be ftp site # Ensure that core repo only depends on core, and the extra repo only @@ -115,7 +116,11 @@ def split_dep_prov_symbol(dep): return (dep, prov, symbol) def repo_from_path(path): - return path.split("/")[-4] + # Community HACK: community still has old + # community/category/pkgname/PKGBUILD path - accomodate for this + if path.split("/")[-1 * (pkgdir_path_depth + 1)] == "community": + return path.split("/")[-1 * (pkgdir_path_depth + 1)] + return path.split("/")[-1 * pkgdir_path_depth] def create_supfile(sourcefile, destfile, newtag): o = open(sourcefile, 'r') @@ -264,8 +269,15 @@ def get_pkgbuilds_in_dir(rootdir): pkgbuild_provides[provide] = pkgname if pkgname != name: mismatches.append(pkgname + " vs. " + fpath.replace(absroot, "")) - if not pkgfound and rootdir.replace(absroot, "").count("/") == 3: - misses.append(rootdir.replace(absroot, "") + "/PKGBUILD") + if not pkgfound and rootdir != absroot: + repo = rootdir.replace(absroot, "").split("/")[1] + num_slashes = pkgdir_path_depth - 1 + # Community HACK: community still has old + # community/category/pkgname/PKGBUILD path - accomodate for this + if repo == "community" and rootdir.replace(absroot, "").count("/") == num_slashes + 1 and rootdir.split("/")[-1] != "CVS": + misses.append(rootdir.replace(absroot, "") + "/PKGBUILD") + if repo != "community" and rootdir.replace(absroot, "").count("/") == num_slashes: + misses.append(rootdir.replace(absroot, "") + "/PKGBUILD") def verify_depends_makedepends(verify_makedeps=False): # Make sure all the deps we parsed are actually packages; also @@ -581,17 +593,13 @@ def print_usage(): print "Usage: check_archlinux [OPTION]" print "" print "Options:" - print " --abs-tree= Check specified tree (assumes the abs tree" - print " is i686 unless overridden with --arch)" - print " --arch= Use specified arch (e.g. 'x86_64')" - print " -g Generate graphical dependency tree(s)" - print " -h, --help Show this help and exit" + print " --abs-tree= REQUIRED Check specified tree (assumes the abs tree" + print " is i686 unless overridden with --arch)" + print " --arch= OPTIONAL Use specified arch (e.g. 'x86_64')" + print " -g OPTIONAL Generate graphical dependency tree(s)" + print " -h, --help OPTIONAL Show this help and exit" print "" print "Examples:" - print "\n Check all arches and do fresh cvs checkouts:" - print " check_archlinux" - print "\n Check x_86_64 only and do fresh cvs checkout:" - print " check_archlinux --arch=x86_64" print "\n Check existing i686 abs tree:" print " check_archlinux --abs-tree=/var/abs" print "\n Check existing x86_64 abs tree and also generate dep tree image:" @@ -631,35 +639,21 @@ if opts != []: self.print_usage() sys.exit() -if len(user_absroot) > 0 and len(user_arch) == 0: +if len(user_absroot) == 0: + self.print_usage() + sys.exit() + +if len(user_arch) == 0: user_arch = valid_archs[0] # i686 default.. if len(user_absroot) > 0: print "Warning: Ensure your ABS tree is clean to prevent false positives." try: - absroots = [] - fsup = 'supfile.curr' for arch in valid_archs: if len(user_arch) == 0 or user_arch == arch: print_heading(arch + " Integrity Check") - if len(user_absroot) == 0: - # Create temp dir for cvs checkout: - absroot = tempfile.mkdtemp(prefix="abs-" + arch + "-") - absroots.append(absroot) - print "\nChecking out clean abs tree" - for repo in include_paths: - print "==>", repo - # Create (and modify) supfile: - if not os.path.exists(abs_conf_dir + '/supfile.' + repo): - print "Cannot find file " + abs_conf_dir + '/supfile.' + repo + ". Aborting..." - sys.exit() - create_supfile(abs_conf_dir + '/supfile.' + repo, fsup, cvs_tags[arch]) - cmd = 'csup -L 0 -r 0 -g -b ' + absroot + ' -c .sup ' + fsup - os.system(cmd) - else: - absroot = user_absroot - absroots.append(absroot) + absroot = user_absroot curr_include_paths = [] for repo in include_paths: curr_include_paths.append(absroot + "/" + repo) @@ -702,23 +696,8 @@ try: print_results() if graphdeptree: visualize_repo() - - print "\nCleaning up temporary abs tree..." - if len(user_absroot) == 0: - # Delete temp abs tree: - if os.path.exists(absroot): - removeall(absroot) - os.rmdir(absroot) + except: - # Cleanup files... - if len(user_absroot) == 0: - print "\nCleaning up temporary abs tree(s)..." - # Delete any temp abs trees: - for path in absroots: - if os.path.exists(path): - removeall(path) - os.rmdir(path) -finally: - # Remove supfile: - if os.path.exists(fsup): - os.remove(fsup) + sys.exit() + +# vim: set ts=2 sw=2 noet : -- cgit v1.2.3-2-g168b From b3120a4c604d4151c8b778f2f4364518e1e66a95 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 20 Jun 2008 14:26:14 -0400 Subject: Remove some 'self' usage from check_archlinux.py Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux.py b/cron-jobs/check_archlinux.py index 055c0a4..61b9bc2 100755 --- a/cron-jobs/check_archlinux.py +++ b/cron-jobs/check_archlinux.py @@ -636,11 +636,11 @@ if opts != []: if a in ("play", "pause", "stop", "next", "prev", "pp", "info", "status", "repeat", "shuffle"): self.single_connect_for_passed_arg(a) else: - self.print_usage() + print_usage() sys.exit() if len(user_absroot) == 0: - self.print_usage() + print_usage() sys.exit() if len(user_arch) == 0: -- cgit v1.2.3-2-g168b From 547b245147b7d44930df46f553067864339f9c43 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 20 Jun 2008 15:19:52 -0400 Subject: Corrections to the devlist-mailer script * Add configurable subject * Add To: and From: headers Signed-off-by: Aaron Griffin --- cron-jobs/devlist-mailer | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer index 094d60d..b24d530 100755 --- a/cron-jobs/devlist-mailer +++ b/cron-jobs/devlist-mailer @@ -4,13 +4,20 @@ LIST="arch-dev-public@archlinux.org" #LIST="aaronmgriffin@gmail.com" -FROM="repomaint" +FROM="repomaint@archlinux.org" + +SUBJECT="Repository Maintenance $(date +"%d-%m-%Y %H:%M")" +if [ $# -eq 1 ]; then + SUBJECT="$1" +fi stdin="$(cat)" #echo used to strip whitespace for checking for actual data if [ -n "$(echo $stdin)" ]; then -echo "Subject: Repository Maintenance $(date +"%d-%m-%Y %H:%M") +echo "Subject: $SUBJECT +To: $LIST +From: $FROM $stdin" | /usr/sbin/sendmail -F$FROM "$LIST" -- cgit v1.2.3-2-g168b From f404689fc11bbeace243779305ede5b7d7270ae8 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 30 Jun 2008 12:43:02 -0400 Subject: Remove community from the integrity check, for now To much output sent to the dev list, this needs to be sent to the aur-general list separately Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux.py b/cron-jobs/check_archlinux.py index 61b9bc2..cb43351 100755 --- a/cron-jobs/check_archlinux.py +++ b/cron-jobs/check_archlinux.py @@ -25,7 +25,8 @@ abs_conf_dir = "/etc/abs" valid_archs = ['i686', 'x86_64'] cvs_tags = {'i686': 'CURRENT', 'x86_64': 'CURRENT-64'} -include_paths = ['core', 'extra', 'community', 'unstable'] +#include_paths = ['core', 'extra', 'community', 'unstable'] +include_paths = ['core', 'extra', 'unstable'] pkgdir_path_depth = 3 base_server = "ftp.archlinux.org" # Must be ftp site -- cgit v1.2.3-2-g168b From 909b017c08109bda405a3e38a59cbf26211e6683 Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Fri, 29 Aug 2008 20:23:26 +0200 Subject: Replace check_archlinux.py by check_archlinux/check_packages.py The old script had several problems so I decided to do a full rewrite. The improvements include : * better and safer parsing of PKGBUILDs It now uses separate parse_pkgbuilds.sh bash script (inspired from namcap) * much better performance A python module for calling vercmp natively, and the algorithm for checking circular dependencies was greatly improved * more accurate dependency and provision handling Now versioned dependencies and provisions are handled correctly. After building the python module and moving it next to the main script, it should be possible to use it like this : For core and extra : ./check_packages.py --abs-tree=/home/abs/rsync/i686 --repos=core,extra ./check_packages.py --abs-tree=/home/abs/rsync/x86_64 --repos=core,extra For community : ./check_packages.py --abs-tree=/home/abs/rsync/i686 --repos=community ./check_packages.py --abs-tree=/home/abs/rsync/x86_64 --repos=community Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux.py | 704 --------------------------- cron-jobs/check_archlinux/README | 8 + cron-jobs/check_archlinux/alpm.c | 40 ++ cron-jobs/check_archlinux/check_packages.py | 392 +++++++++++++++ cron-jobs/check_archlinux/parse_pkgbuilds.sh | 78 +++ cron-jobs/check_archlinux/setup.py | 10 + 6 files changed, 528 insertions(+), 704 deletions(-) delete mode 100755 cron-jobs/check_archlinux.py create mode 100644 cron-jobs/check_archlinux/README create mode 100644 cron-jobs/check_archlinux/alpm.c create mode 100755 cron-jobs/check_archlinux/check_packages.py create mode 100755 cron-jobs/check_archlinux/parse_pkgbuilds.sh create mode 100644 cron-jobs/check_archlinux/setup.py (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux.py b/cron-jobs/check_archlinux.py deleted file mode 100755 index cb43351..0000000 --- a/cron-jobs/check_archlinux.py +++ /dev/null @@ -1,704 +0,0 @@ -#!/usr/bin/env python -# -# check_archlinux.py -# -# Written by Scott Horowitz -# Graphical dependency tree by Cesar G. Miguel -# -# This script currently checks for a number of issues in your ABS tree: -# 1. Directories with missing PKGBUILDS -# 2. Duplicate PKGBUILDs -# 3. Missing (make-)dependencies, taking into account 'provides' -# 4. Provisioned dependencies -# 5. Circular dependencies -# 6. Valid arch's in PKGBUILDS -# 7. Missing packages in Arch repos -# 8. PKGBUILD names that don't match their directory -# 9. Hierarchy of repos (e.g., that a core package doesn't depend on -# a non-core package) -# It can also, optionally, generate a graphical representation of the -# dependency tree. -# -# Todo: -# 1. Accommodate testing repo? - -abs_conf_dir = "/etc/abs" -valid_archs = ['i686', 'x86_64'] -cvs_tags = {'i686': 'CURRENT', 'x86_64': 'CURRENT-64'} -#include_paths = ['core', 'extra', 'community', 'unstable'] -include_paths = ['core', 'extra', 'unstable'] -pkgdir_path_depth = 3 - -base_server = "ftp.archlinux.org" # Must be ftp site -# Ensure that core repo only depends on core, and the extra repo only -# depends on core and extra, etc. -repo_hierarchy = {'core': ('core'), \ -'extra': ('core', 'extra'), \ -'unstable': ('core', 'extra', 'unstable'), \ -'community': ('core', 'extra', 'unstable', 'community')} - -pkgname_str = "pkgname=" -dep_str = "depends=(" -makedep_str = "makedepends=(" -provides_str = "provides=(" -arch_str = "arch=(" -pkgver_str = "pkgver=" -pkgrel_str = "pkgrel=" -build_str = "build()" -source_str = "source=" -url_str = "url=" -mathlist = (">=", "<=", "=", ">", "<") -sup_tag_str = "*default tag=" - -import os, sys, getopt, tempfile -from ftplib import FTP -try: - import pydot - HAS_PYDOT = True -except: - HAS_PYDOT = False - -def print_heading(heading): - print "" - print "=" * (len(heading) + 4) - print "= " + heading + " =" - print "=" * (len(heading) + 4) - -def print_subheading(subheading): - print "" - print subheading - print "-" * (len(subheading) + 2) - -def rmgeneric(path, __func__): - try: - __func__(path) - except OSError, (errno, strerror): - pass - -def removeall(path): - if not os.path.isdir(path): - return - - files=os.listdir(path) - - for x in files: - fullpath=os.path.join(path, x) - if os.path.isfile(fullpath): - f=os.remove - rmgeneric(fullpath, f) - elif os.path.isdir(fullpath): - removeall(fullpath) - f=os.rmdir - rmgeneric(fullpath, f) - -def update_var(line, user_vars, pkgpath): - if line.count("$") > 0: - export_line = "" - for var in user_vars: - if line.count(var[0]) > 0: - export_line = export_line + var[0] + "=" + var[1] + " && " - if line.startswith("(") and line.endswith(")"): - line = line[1:-1] - export_line = export_line + "echo \"" + line + "\"" - line = os.popen(export_line).read().replace("\n", "") - return line - -def split_dep_prov_symbol(dep): - # Splits 'foo>=1.2.3' into ('foo', '1.2.3', '>=') - prov = "" - symbol = "" - for char in mathlist: - pos = dep.find(char) - if pos > -1: - prov = dep[pos:].replace(char, "") - dep = dep[:pos] - symbol = char - break - return (dep, prov, symbol) - -def repo_from_path(path): - # Community HACK: community still has old - # community/category/pkgname/PKGBUILD path - accomodate for this - if path.split("/")[-1 * (pkgdir_path_depth + 1)] == "community": - return path.split("/")[-1 * (pkgdir_path_depth + 1)] - return path.split("/")[-1 * pkgdir_path_depth] - -def create_supfile(sourcefile, destfile, newtag): - o = open(sourcefile, 'r') - info = o.read() - o.close() - lines = info.split("\n") - - o = open(destfile, 'w') - for line in lines: - line = line.strip() - if line[:len(sup_tag_str)] == sup_tag_str: - line = sup_tag_str + newtag - o.write(line + "\n") - o.close() - -def get_deps_provides_etc(pkgpath): - # Parse PKGBUILD for name, depends, makedepends, provides, arch's, and version - o = open(pkgpath, 'r') - info = o.read() - o.close() - lines = info.split("\n") - - deps = [] - provides = [] - archs = [] - makedeps = [] - array = [] - user_vars = [] - continue_line = False - for line in lines: - line = line.strip() - if line.find("#") > -1: - line = line[:line.find("#")].strip() - if not continue_line: - deps_line = False - provides_line = False - arch_line = False - makedeps_line = False - if line[:len(dep_str)] == dep_str: - line = line.replace(dep_str,"") - deps_line = True - elif line[:len(makedep_str)] == makedep_str: - line = line.replace(makedep_str,"") - makedeps_line = True - elif line[:len(provides_str)] == provides_str: - line = line.replace(provides_str,"") - provides_line = True - elif line[:len(arch_str)] == arch_str: - line = line.replace(arch_str, "") - arch_line = True - elif line[:len(pkgname_str)] == pkgname_str: - pkgname = line.replace(pkgname_str, "") - if pkgname.startswith("\"") and pkgname.endswith("\""): - pkgname = pkgname[1:-1] - pkgname = update_var(pkgname, user_vars, pkgpath) - user_vars.append([pkgname_str, pkgname]) - line = "" - elif line[:len(pkgver_str)] == pkgver_str: - pkgver = line.replace(pkgver_str, "") - if pkgver.startswith("\"") and pkgver.endswith("\""): - pkgver = pkgver[1:-1] - pkgver = update_var(pkgver, user_vars, pkgpath) - user_vars.append([pkgver_str[:-1], pkgver]) - line = "" - elif line[:len(pkgrel_str)] == pkgrel_str: - pkgrel = line.replace(pkgrel_str, "") - if pkgrel.startswith("\"") and pkgrel.endswith("\""): - pkgrel = pkgrel[1:-1] - pkgrel = update_var(pkgrel, user_vars, pkgpath) - user_vars.append([pkgrel_str[:-1], pkgrel]) - line = "" - elif line[:len(build_str)] == build_str: - break - elif not continue_line: - if line.count("=") == 1 and line.count(" ") == 0 and line[:1] != "#" and \ - line[:len(source_str)] != source_str and line[:len(url_str)] != url_str: - split = line.split("=") - for item in range(len(split)): - split[item] = update_var(split[item], user_vars, pkgpath) - user_vars.append(split) - line = "" - if len(line) > 0: - pos = line.find(")") - if pos > -1: - # strip everything from closing paranthesis on - # since some PKGBUILDS have comments after the - # depends array - line = line[:pos] - line = line.split(' ') - for i in range(len(line)): - line[i] = line[i].replace("'","").replace('"','') - line[i] = update_var(line[i], user_vars, pkgpath) - if len(line[i]) > 0: - if deps_line: - deps.append(line[i]) - array=deps - elif provides_line: - provides.append(line[i]) - array=provides - elif arch_line: - archs.append(line[i]) - array=archs - elif makedeps_line: - makedeps.append(line[i]) - array=makedeps - if array and (array[-1] == "\\" or array[-1][-1] == "\\"): - # continue reading deps/provides on next line - if array[-1] == "\\": - array.pop(-1) - else: - array[-1] = array[-1].replace("\\", "") - continue_line = True - else: - continue_line = False - version = pkgver + "-" + pkgrel - return (pkgname, deps, makedeps, provides, archs, version) - -def get_pkgbuilds_in_dir(rootdir): - # Recursively populates pkgbuild_deps, pkgbuild_paths, etc. - # dicts with info from each PKGBUILD found in rootdir: - if rootdir != absroot: - if rootdir.count("/") == (absroot.count("/")+1) and rootdir not in curr_include_paths: - return - pkgfound = False - for f in os.listdir(rootdir): - fpath = rootdir + "/" + f - if os.path.isdir(fpath): - get_pkgbuilds_in_dir(fpath) - elif f == 'PKGBUILD': - pkgfound = True - name = rootdir.split("/")[-1] - if name in pkgbuild_deps: - dups.append(fpath.replace(absroot, "") + " vs. " + pkgbuild_paths[name].replace(absroot, "")) - else: - (pkgname, deps, makedeps, provides, archs, version) = get_deps_provides_etc(fpath) - pkgbuild_deps[pkgname] = deps - pkgbuild_makedeps[pkgname] = makedeps - pkgbuild_paths[pkgname] = fpath - pkgbuild_archs[pkgname] = archs - pkgbuild_versions[pkgname] = version - # We'll store the provides "backwards" compared to - # the other dicts. This will make searching for - # provides easier by being able to look up the - # provide name itself and find what provides it - for provide in provides: - pkgbuild_provides[provide] = pkgname - if pkgname != name: - mismatches.append(pkgname + " vs. " + fpath.replace(absroot, "")) - if not pkgfound and rootdir != absroot: - repo = rootdir.replace(absroot, "").split("/")[1] - num_slashes = pkgdir_path_depth - 1 - # Community HACK: community still has old - # community/category/pkgname/PKGBUILD path - accomodate for this - if repo == "community" and rootdir.replace(absroot, "").count("/") == num_slashes + 1 and rootdir.split("/")[-1] != "CVS": - misses.append(rootdir.replace(absroot, "") + "/PKGBUILD") - if repo != "community" and rootdir.replace(absroot, "").count("/") == num_slashes: - misses.append(rootdir.replace(absroot, "") + "/PKGBUILD") - -def verify_depends_makedepends(verify_makedeps=False): - # Make sure all the deps we parsed are actually packages; also - # ensure we meet dep provisions. - if verify_makedeps: - array = pkgbuild_makedeps - else: - array = pkgbuild_deps - for pkgname in array: - deps = array[pkgname] - pkg_repo = repo_from_path(pkgbuild_paths[pkgname]) - deps_to_pop = [] - for i in range(len(deps)): - dep = deps[i] - (dep, prov, char) = split_dep_prov_symbol(dep) - try: - x = pkgbuild_deps[dep] - # Check that prov is met too: - if len(prov) > 0: - compare_str = "vercmp " + pkgbuild_versions[dep] + " " + prov - error = False - if char == "<=": - if int(os.popen(compare_str).read().replace("\n", "")) > 0: - error = True - elif char == ">=": - if int(os.popen(compare_str).read().replace("\n", "")) < 0: - error = True - elif char == "=": - if int(os.popen(compare_str).read().replace("\n", "")) != 0: - error = True - elif char == ">": - if int(os.popen(compare_str).read().replace("\n", "")) <= 0: - error = True - elif char == "<": - if int(os.popen(compare_str).read().replace("\n", "")) >= 0: - error = True - if error: - if verify_makedeps: - unmet_makedep_provs.append(pkgname + " --> '" + dep + char + prov + "'") - else: - unmet_dep_provs.append(pkgname + " --> '" + dep + char + prov + "'") - # Check repos fit hierarchy scheme: - dep_repo = repo_from_path(pkgbuild_paths[dep]) - try: - valid_repos = repo_hierarchy[pkg_repo] - # Make sure dep repo is one of the valid repos: - if dep_repo not in valid_repos: - if verify_makedeps: - makedep_hierarchy.append(pkg_repo + "/" + pkgname + " depends on " + dep_repo + "/" + dep) - else: - dep_hierarchy.append(pkg_repo + "/" + pkgname + " depends on " + dep_repo + "/" + dep) - except: - pass - except: - # Check if a package provides this dep: - try: - x = pkgbuild_provides[dep] - except: - if verify_makedeps: - missing_makedeps.append(pkgname + " --> '" + dep + "'") - else: - missing_deps.append(pkgname + " --> '" + dep + "'") - deps_to_pop.append(i) - # Pop deps not found from end to beginning: - while len(deps_to_pop) > 0: - deps.pop(deps_to_pop[-1]) - deps_to_pop.pop(-1) - -def verify_archs(): - for pkgname in pkgbuild_archs: - newarch = [] - archs = pkgbuild_archs[pkgname] - for arch in archs: - if arch not in valid_archs: - invalid_archs.append(pkgname + " --> " + arch) - else: - newarch.append(arch) - if len(newarch) > 0: - pkgbuild_archs[pkgname] = newarch - -def verify_packages(tree_arch): - # Make sure packages exist in Arch repo(s): - ftp = FTP(base_server) - ftp.login() - prev_wd = "" - # Find all repos/archs; marching through them in order will greatly speed - # up searching for ftp files by minimizing the number of .nlst's that we - # have to do. - repos = [] - for pkgname in pkgbuild_paths: - pkgrepo = repo_from_path(pkgbuild_paths[pkgname]) - if not pkgrepo in repos: - repos.append(pkgrepo) - archs = [] - for pkgname in pkgbuild_archs: - pkgarchs = pkgbuild_archs[pkgname] - for arch in pkgarchs: - if not arch in archs: - archs.append(arch) - for r in repos: - for pkgname in pkgbuild_archs: - repo = repo_from_path(pkgbuild_paths[pkgname]) - if repo == r: - archs = pkgbuild_archs[pkgname] - pkgver_rel = pkgbuild_versions[pkgname] - for arch in archs: - if arch == tree_arch: - # Check for file: - wd = repo + "/os/" + arch - if wd != prev_wd: - ftpfiles = ftp.nlst(wd) - prev_wd = wd - fname_new = wd + "/" + pkgname + "-" + pkgver_rel + "-" + arch + ".pkg.tar.gz" - fname_old = wd + "/" + pkgname + "-" + pkgver_rel + ".pkg.tar.gz" - if fname_old not in ftpfiles and fname_new not in ftpfiles: - missing_pkgs.append(pkgname + "-" + pkgver_rel + " in " + wd) - ftp.quit() - -def subset_superset_path(currpath, currpathnum, paths): - # If a pair of subset/superset paths are found, - # pop the superset one to show the more minimal - # case. - # - # e.g. foo > bar > baz > foo (superset) - # foo > bar > foo (subset) - # --> pop the superset - # - currdeps = currpath.split(">") - currdeps = list(set(currdeps)) - currdeps.sort() - pathnum = 0 - for path in paths: - if pathnum != currpathnum: - deps = path.split(">") - deps = list(set(deps)) - deps.sort() - if len(currdeps) < len(path): - subset = True - for d in currdeps: - if not d in path: - subset = False - break - if subset: - circular_deps.pop(pathnum) - if pathnum <= currpathnum: - currpathnum -= 1 - elif len(currdeps) > len(path): - superset = True - for d in path: - if not d in currdeps: - superset = False - break - if superset: - circular_deps.pop(currpathnum) - currpathnum -= 1 - pathnum += 1 - return True - -def unique_path(currpath, paths): - # Returns false if an equivalent path exists in - # paths. - # - # e.g. foo > bar > foo - # bar > foo > bar - # - currdeps = currpath.split(">") - currdeps = list(set(currdeps)) - currdeps.sort() - for path in paths: - deps = path.split(">") - deps = list(set(deps)) - deps.sort() - if currdeps == deps: - return False - return True - -def update_paths(paths, dep, dep2): - # Update paths by appending new paths with dep2 - # based on all the different ways we could get - # to dep2. Returns True if a path was updated. - new_path = False - for i in range(len(paths)): - array = paths[i].split(">") - newpath = paths[i] + ">" + dep2 - if array[-1] == dep and not dep2 in array and unique_path(newpath, paths): - paths.append(newpath) - new_path = True - return new_path - -def check_name_in_recursive_deps(pkgname): - # Retrieve all recursive dependencies from a package and - # determines if pkgname is found in one of its deps. - recursive_deps = [] - for dep in pkgbuild_deps[pkgname]: - dep = split_dep_prov_symbol(dep)[0] # Strip any provision - recursive_deps.append(dep) - paths = [] - for dep in recursive_deps: - dep = split_dep_prov_symbol(dep)[0] # Strip any provision - paths.append(dep) - searching = True - while searching: - searching = False - for dep in recursive_deps: - for dep2 in pkgbuild_deps[dep]: - dep2 = split_dep_prov_symbol(dep2)[0] # Strip any provision - # This is a HUGE time-saver. Instead of generating every single - # possible path that can yield a circular dep, we'll reduce - # the number greatly by throwing out paths like such: - # - # If we have a path: foo>bar>baz>blah>poop>foo - # We will throw out any: - # foo>...>bar>baz>blah>poop>foo - # foo>...>baz>blah>poop>foo - # foo>...>blah>poop>foo - # and so on. Otherwise we will find hundreds or even thousands - # of possible paths that all essentially represent the same - # circular dep. - # - # However, we will always let pkgname through in order to make - # sure we can find multiple circular deps for a given pkg. - if dep2 not in recursive_deps or dep2 == pkgname: - updated = update_paths(paths, dep, dep2) - if dep2 not in recursive_deps: - recursive_deps.append(dep2) - if updated: - searching = True - # Done searching, store circular deps: - for path in paths: - if path.split(">")[-1] == pkgname: - if unique_path(pkgname + ">" + path, circular_deps): - circular_deps.append(pkgname + ">" + path) - # Reduce any subset/superset path pairs: - pathnum = 0 - for path in circular_deps: - subset_superset_path(path, pathnum, circular_deps) - pathnum += 1 - -def circular_deps_check(): - # Check for circular dependencies: - for pkgname in pkgbuild_deps: - check_name_in_recursive_deps(pkgname) - -def visualize_repo(): - output = 'digraph G { \n \ - concentrate = true; \n \ - ordering = out; \n \ - ranksep=5.0; \n \ - node [style=filled,fontsize=8]; \n' - - # draws circular dependencies in red - for path in circular_deps: - output += '\t "'+path[0]+'"' - deps = path.split(">") - for d in deps: - output += ' -> "'+d+'"' - output += ' [color=red]\n' - - for pkg in pkgbuild_deps.keys(): - output += '\t "'+pkg+'" -> { ' - for d in pkgbuild_deps[pkg]: - d = split_dep_prov_symbol(d)[0] # Strip any provision - output += '"'+d+'"; ' - output += '}\n' - - output += '}' - - # Uncomment these lines to get a file dump called - # 'output'. This can be used to manually generate - # an image using, e.g., dot -Tsvg output tree.svg - #dump = open('output', 'w') - #dump.write(output) - #dump.close() - - fname = 'dependency_tree-' + arch + '.svg' - print "Generating " + fname + "..." - g = pydot.graph_from_dot_data(output) - g.write(fname, prog='dot', format='svg') - -def print_result(list, subheading): - if len(list) > 0: - print_subheading(subheading) - for item in list: - print item - -def print_results(): - print_result(misses, "Missing PKGBUILDs") - print_result(mismatches, "Mismatched Pkgnames") - print_result(dups, "Duplicate PKGBUILDs") - print_result(missing_deps, "Missing Dependencies") - print_result(missing_makedeps, "Missing Makedepends") - print_result(unmet_dep_provs, "Unmet Dependency Provisions") - print_result(unmet_makedep_provs, "Unmet Makedepends Provisions") - print_result(dep_hierarchy, "Repo Hierarchy for Dependencies") - print_result(makedep_hierarchy, "Repo Hierarchy for Makedepends") - print_result(invalid_archs, "Invalid Archs") - print_result(circular_deps, "Circular Dependencies") - print_result(missing_pkgs, "Missing Repo Packages") - print_subheading("Summary") - print "Dirs with missing PKGBUILDs: ", len(misses) - print "Duplicate PKGBUILDs: ", len(dups) - print "Missing (make)dependencies: ", len(missing_deps)+len(missing_makedeps) - print "Unmet provisioned (make)dependencies: ", len(unmet_dep_provs)+len(unmet_makedep_provs) - print "Circular dependencies: ", len(circular_deps) - print "Invalid archs: ", len(invalid_archs) - print "Missing packages in repos: ", len(missing_pkgs) - print "Mismatching PKGBUILD names: ", len(mismatches) - print "Repo hierarchy problems: ", len(dep_hierarchy)+len(makedep_hierarchy) - print "" - -def print_usage(): - print "" - print "Usage: check_archlinux [OPTION]" - print "" - print "Options:" - print " --abs-tree= REQUIRED Check specified tree (assumes the abs tree" - print " is i686 unless overridden with --arch)" - print " --arch= OPTIONAL Use specified arch (e.g. 'x86_64')" - print " -g OPTIONAL Generate graphical dependency tree(s)" - print " -h, --help OPTIONAL Show this help and exit" - print "" - print "Examples:" - print "\n Check existing i686 abs tree:" - print " check_archlinux --abs-tree=/var/abs" - print "\n Check existing x86_64 abs tree and also generate dep tree image:" - print " check_archlinux --abs-tree=/var/abs --arch=x86_64 -g" - print "" - -graphdeptree = False -user_absroot = "" -user_arch = "" -try: - opts, args = getopt.getopt(sys.argv[1:], "g", ["abs-tree=", "arch="]) -except getopt.GetoptError: - print_usage() - sys.exit() -if opts != []: - for o, a in opts: - if o in ("-g"): - graphdeptree = True - if not HAS_PYDOT: - print "You must install pydot to generate a graphical dependency tree. Aborting..." - sys.exit() - elif o in ("--abs-tree"): - user_absroot = a - elif o in ("--arch"): - user_arch = a - if user_arch not in valid_archs: - print "You did not specify a valid arch. Aborting..." - sys.exit() - else: - print_usage() - sys.exit() - if args != []: - for a in args: - if a in ("play", "pause", "stop", "next", "prev", "pp", "info", "status", "repeat", "shuffle"): - self.single_connect_for_passed_arg(a) - else: - print_usage() - sys.exit() - -if len(user_absroot) == 0: - print_usage() - sys.exit() - -if len(user_arch) == 0: - user_arch = valid_archs[0] # i686 default.. - -if len(user_absroot) > 0: - print "Warning: Ensure your ABS tree is clean to prevent false positives." - -try: - for arch in valid_archs: - if len(user_arch) == 0 or user_arch == arch: - print_heading(arch + " Integrity Check") - absroot = user_absroot - curr_include_paths = [] - for repo in include_paths: - curr_include_paths.append(absroot + "/" + repo) - - # Re-init vars for new abs tree: - pkgbuild_deps = {} # pkgname: [dep1, dep2>foo, dep3=bar, ...] - pkgbuild_makedeps = {} # pkgname: [dep1, dep2>foo, dep3=bar, ...] - pkgbuild_provides = {} # provide_name: pkgname - pkgbuild_paths = {} # pkgname: /var/abs/foo/bar/pkgname/PKGBUILD - pkgbuild_archs = {} # pkgname: [i686, x86_64] - pkgbuild_versions = {} # pkname: 1.0.4-2 - # circular_deps is not a dict to accommodate multiple circ deps for a given pkg - circular_deps = [] # pkgname>dep1>dep2>...>pkgname - mismatches = [] - misses = [] - dups = [] - missing_deps = [] - unmet_dep_provs = [] - dep_hierarchy = [] - missing_makedeps = [] - unmet_makedep_provs = [] - makedep_hierarchy = [] - invalid_archs = [] - missing_pkgs = [] - - # Verify stuff for abs tree: - print "\nPerforming integrity checks..." - print "==> parsing pkgbuilds" - get_pkgbuilds_in_dir(absroot) - print "==> checking dependencies" - verify_depends_makedepends() - print "==> checking makedepends" - verify_depends_makedepends(True) - print "==> checking archs" - verify_archs() - print "==> checking for circular dependencies" - circular_deps_check() - print "==> checking repo packages" - verify_packages(arch) - print_results() - if graphdeptree: - visualize_repo() - -except: - sys.exit() - -# vim: set ts=2 sw=2 noet : diff --git a/cron-jobs/check_archlinux/README b/cron-jobs/check_archlinux/README new file mode 100644 index 0000000..f3a1b90 --- /dev/null +++ b/cron-jobs/check_archlinux/README @@ -0,0 +1,8 @@ +1) Build the python module +$ python setup.py build + +2) copy it back to the current working directory +$ cp build/lib.*/alpm.* . + +3) run the script +$ ./check_packages.py -h diff --git a/cron-jobs/check_archlinux/alpm.c b/cron-jobs/check_archlinux/alpm.c new file mode 100644 index 0000000..0b7cd2c --- /dev/null +++ b/cron-jobs/check_archlinux/alpm.c @@ -0,0 +1,40 @@ +#include +#include + +static PyObject * +alpm_vercmp(PyObject *self, PyObject *args) +{ + const char *v1, *v2; + int ret; + + if (!PyArg_ParseTuple(args, "ss", &v1, &v2)) + return NULL; + ret = alpm_pkg_vercmp(v1, v2); + return Py_BuildValue("i", ret); +} + +static PyMethodDef AlpmMethods[] = { + {"vercmp", alpm_vercmp, METH_VARARGS, + "Execute vercmp."}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +PyMODINIT_FUNC +initalpm(void) +{ + (void) Py_InitModule("alpm", AlpmMethods); +} + +int +main(int argc, char *argv[]) +{ + /* Pass argv[0] to the Python interpreter */ + Py_SetProgramName(argv[0]); + + /* Initialize the Python interpreter. Required. */ + Py_Initialize(); + + /* Add a static module */ + initalpm(); + return 0; +} diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py new file mode 100755 index 0000000..2d2efbe --- /dev/null +++ b/cron-jobs/check_archlinux/check_packages.py @@ -0,0 +1,392 @@ +#!/usr/bin/python +# +# check_archlinux.py +# +# Original script by Scott Horowitz +# Rewritten by Xavier Chantry +# +# This script currently checks for a number of issues in your ABS tree: +# 1. Directories with missing PKGBUILDS +# 2. Invalid PKGBUILDs (bash syntax error for instance) +# 3. PKGBUILD names that don't match their directory +# 4. Duplicate PKGBUILDs +# 5. Valid arch's in PKGBUILDS +# 6. Missing (make-)dependencies +# 7. Hierarchy of repos (e.g., that a core package doesn't depend on +# a non-core package) +# 8. Circular dependencies + +import os,re,commands,getopt,sys,alpm +import pdb + +packages = {} # pkgname : PacmanPackage +provisions = {} # provision : PacmanPackage +pkgdeps,makepkgdeps = {},{} # pkgname : list of the PacmanPackage dependencies +invalid_pkgbuilds = [] +missing_pkgbuilds = [] +dups = [] + +mismatches = [] +missing_deps = [] +missing_makedeps = [] +invalid_archs = [] +dep_hierarchy = [] +makedep_hierarchy = [] +circular_deps = [] # pkgname>dep1>dep2>...>pkgname +checked_deps = [] + +class PacmanPackage: + def __init__(self): + self.name,self.version = "","" + self.path,self.repo = "","" + self.deps,self.makedeps = [],[] + self.provides,self.conflicts = [],[] + self.archs = [] + +class Depend: + def __init__(self,name,version,mod): + self.name = name + self.version = version + self.mod = mod + +def parse_pkgbuilds(repos): + oldcwd = os.getcwd() + os.chdir(absroot) + for repo in repos: + data = commands.getoutput(oldcwd + '/parse_pkgbuilds.sh ' + repo) + parse_data(repo,data) + os.chdir(oldcwd) + +def parse_data(repo,data): + attrname = None + + for line in data.split('\n'): + if line.startswith('%'): + attrname = line.strip('%').lower() + elif line.strip() == '': + attrname = None + elif attrname == "invalid": + if repo in repos: + invalid_pkgbuilds.append(line) + elif attrname == "missing": + if repo in repos: + missing_pkgbuilds.append(line) + elif attrname == "name": + pkg = PacmanPackage() + pkg.name = line + pkg.repo = repo + dup = None + if packages.has_key(pkg.name): + dup = packages[pkg.name] + packages[pkg.name] = pkg + elif attrname == "version": + pkg.version = line + elif attrname == "path": + pkg.path = line + if dup != None and (pkg.repo in repos or dup.repo in repos): + dups.append(pkg.path + " vs. " + dup.path) + elif attrname == "arch": + pkg.archs.append(line) + elif attrname == "depends": + pkg.deps.append(line) + elif attrname == "makedepends": + pkg.makedeps.append(line) + elif attrname == "conflicts": + pkg.conflicts.append(line) + elif attrname == "provides": + pkg.provides.append(line) + provname=line.split("=")[0] + if not provisions.has_key(provname): + provisions[provname] = [] + provisions[provname].append(pkg) + +def splitdep(dep): + name = dep + version = "" + mod = "" + for char in (">=", "<=", "=", ">", "<"): + pos = dep.find(char) + if pos > -1: + name = dep[:pos] + version = dep[pos:].replace(char, "") + mod = char + break + return Depend(name,version,mod) + +def splitprov(prov): + name = prov + version = "" + pos = prov.find("=") + if pos > -1: + name = prov[:pos] + version = prov[pos:].replace("=", "") + return (name,version) + +def vercmp(v1,mod,v2): + res = alpm.vercmp(v1,v2) + if res == 0: + return (mod.find("=") > -1) + elif res < 0: + return (mod.find("<") > -1) + elif res > 0: + return (mod.find(">") > -1) + return False + + +def depcmp(name,version,dep): + if name != dep.name: + return False + if dep.version == "" or dep.mod == "": + return True + if version == "": + return False + return vercmp(version,dep.mod,dep.version) + +def provcmp(pkg,dep): + for prov in pkg.provides: + (provname,provver) = splitprov(prov) + if depcmp(provname,provver,dep): + return True + return False + +def verify_dep(dep): + dep = splitdep(dep) + if packages.has_key(dep.name): + pkg = packages[dep.name] + if depcmp(pkg.name,pkg.version,dep): + return [pkg] + if provisions.has_key(dep.name): + provlist = provisions[dep.name] + results = [] + for prov in provlist: + if provcmp(prov,dep): + results.append(prov) + return results + return [] + +def verify_deps(name,repo,deps): + pkg_deps = [] + missdeps = [] + hierarchy = [] + for dep in deps: + pkglist = verify_dep(dep) + if pkglist == []: + missdeps.append(name + " --> '" + dep + "'") + else: + valid_repos = get_repo_hierarchy(repo) + pkgdep = None + for pkg in pkglist: + if pkg.repo in valid_repos: + pkgdep = pkg + break + if not pkgdep: + pkgdep = pkglist[0] + hierarchy.append(repo + "/" + name + " depends on " + pkgdep.repo + "/" + pkgdep.name) + pkg_deps.append(pkgdep) + + return (pkg_deps,missdeps,hierarchy) + +def get_repo_hierarchy(repo): + repo_hierarchy = {'core': ['core'], \ + 'extra': ['core', 'extra'], \ + 'community': ['core', 'extra', 'community']} + if repo_hierarchy.has_key(repo): + return repo_hierarchy[repo] + else: + return ['core','extra','community'] + +def verify_archs(name,archs): + valid_archs = ['i686', 'x86_64'] + invalid_archs = [] + for arch in archs: + if arch not in valid_archs: + invalid_archs.append(name + " --> " + arch) + return invalid_archs + +def find_scc(packages): + # reset all variables + global index,S,pkgindex,pkglowlink + index = 0 + S = [] + pkgindex = {} + pkglowlink = {} + cycles = [] + for pkg in packages: + tarjan(pkg) + +def tarjan(pkg): + global index,S,pkgindex,pkglowlink,cycles + pkgindex[pkg] = index + pkglowlink[pkg] = index + index += 1 + checked_deps.append(pkg) + S.append(pkg) + if pkgdeps.has_key(pkg): + deps = pkgdeps[pkg] + else: + print pkg.name + deps = [] + for dep in deps: + if not pkgindex.has_key(dep): + tarjan(dep) + pkglowlink[pkg] = min(pkglowlink[pkg],pkglowlink[dep]) + elif dep in S: + pkglowlink[pkg] = min(pkglowlink[pkg],pkgindex[dep]) + if pkglowlink[pkg] == pkgindex[pkg]: + dep = S.pop() + if pkg == dep: + return + path = pkg.name + while pkg != dep: + path = dep.name + ">" + path + dep = S.pop() + path = dep.name + ">" + path + if pkg.repo in repos: + circular_deps.append(path) + +def print_heading(heading): + print "" + print "=" * (len(heading) + 4) + print "= " + heading + " =" + print "=" * (len(heading) + 4) + +def print_subheading(subheading): + print "" + print subheading + print "-" * (len(subheading) + 2) + +def print_missdeps(pkgname,missdeps) : + for d in missdeps: + print pkgname + " : " + d + +def print_result(list, subheading): + if len(list) > 0: + print_subheading(subheading) + for item in list: + print item + +def print_results(): + print_result(missing_pkgbuilds, "Missing PKGBUILDs") + print_result(invalid_pkgbuilds, "Invalid PKGBUILDs") + print_result(mismatches, "Mismatched Pkgnames") + print_result(dups, "Duplicate PKGBUILDs") + print_result(invalid_archs, "Invalid Archs") + print_result(missing_deps, "Missing Dependencies") + print_result(missing_makedeps, "Missing Makedepends") + print_result(dep_hierarchy, "Repo Hierarchy for Dependencies") + print_result(makedep_hierarchy, "Repo Hierarchy for Makedepends") + print_result(circular_deps, "Circular Dependencies") + print_subheading("Summary") + print "Missing PKGBUILDs: ", len(missing_pkgbuilds) + print "Invalid PKGBUILDs: ", len(invalid_pkgbuilds) + print "Mismatching PKGBUILD names: ", len(mismatches) + print "Duplicate PKGBUILDs: ", len(dups) + print "Invalid archs: ", len(invalid_archs) + print "Missing (make)dependencies: ", len(missing_deps)+len(missing_makedeps) + print "Repo hierarchy problems: ", len(dep_hierarchy)+len(makedep_hierarchy) + print "Circular dependencies: ", len(circular_deps) + print "" + +def print_usage(): + print "" + print "Usage: ./check_packages.py [OPTION]" + print "" + print "Options:" + print " --abs-tree= Check specified tree (default : /var/abs)" + print " --repos= Check specified repos (default : core,extra)" + print " -h, --help Show this help and exit" + print "" + print "Examples:" + print "\n Check core and extra in existing abs tree:" + print " ./check_packages.py --abs-tree=/var/abs --repos=core,extra" + print "\n Check community:" + print " ./check_packages.py --abs-tree=/var/abs --repos=community" + print "" + +## Default path to the abs root directory +absroot = "/var/abs" +## Default list of repos to check +repos = ['core', 'extra'] + +try: + opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos="]) +except getopt.GetoptError: + print_usage() + sys.exit() +if opts != []: + for o, a in opts: + if o in ("--abs-tree"): + absroot = a + elif o in ("--repos"): + repos = a.split(",") + else: + print_usage() + sys.exit() + if args != []: + print_usage() + sys.exit() + +if not os.path.isdir(absroot): + print "Error : the abs tree " + absroot + " does not exist" + sys.exit() +for repo in repos: + repopath = absroot + "/" + repo + if not os.path.isdir(repopath): + print "Error : the repository " + repo + " does not exist in " + absroot + sys.exit() +# repos which need to be loaded +loadrepos = set([]) +for repo in repos: + loadrepos = loadrepos | set(get_repo_hierarchy(repo)) + +print_heading("Integrity Check") +print "\nPerforming integrity checks..." + +print "==> parsing pkgbuilds" +parse_pkgbuilds(loadrepos) + +repopkgs = {} +for name,pkg in packages.iteritems(): + if pkg.repo in repos: + repopkgs[name] = pkg + +print "==> checking mismatches" +for name,pkg in repopkgs.iteritems(): + pkgdirname = pkg.path.split("/")[-1] + if name != pkgdirname: + mismatches.append(name + " vs. " + pkg.path) + +print "==> checking archs" +for name,pkg in repopkgs.iteritems(): + archs = verify_archs(name,pkg.archs) + invalid_archs.extend(archs) + +# ugly hack to strip the weird kblic- deps +for name,pkg in packages.iteritems(): + p = re.compile('klibc-[A-Za-z0-9]{20,}|klibc-\*') + pkg.deps = [dep for dep in pkg.deps if not p.match(dep)] + pkg.makedeps = [dep for dep in pkg.makedeps if not p.match(dep)] + +print "==> checking dependencies" +for name,pkg in repopkgs.iteritems(): + (deps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.deps) + pkgdeps[pkg] = deps + missing_deps.extend(missdeps) + dep_hierarchy.extend(hierarchy) + +print "==> checking makedepends" +for name,pkg in repopkgs.iteritems(): + (makedeps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.makedeps) + makepkgdeps[pkg] = makedeps + missing_makedeps.extend(missdeps) + makedep_hierarchy.extend(hierarchy) + +print "==> checking for circular dependencies" +# make sure pkgdeps is filled for every package +for name,pkg in packages.iteritems(): + if not pkgdeps.has_key(pkg): + (deps,missdeps,_) = verify_deps(name,pkg.repo,pkg.deps) + pkgdeps[pkg] = deps +find_scc(repopkgs.values()) + +print_results() diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh new file mode 100755 index 0000000..d4205ae --- /dev/null +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +parse() { + unset pkgname pkgver pkgrel + unset depends makedepends conflicts provides + ret=0 + dir=$1 + pkgbuild=$dir/PKGBUILD + source $pkgbuild &>/dev/null || ret=$? + + # ensure $pkgname and $pkgver variables were found + if [ $ret -ne 0 -o -z "$pkgname" -o -z "$pkgver" ]; then + echo -e "%INVALID%\n$pkgbuild\n" + return 1 + fi + + echo -e "%NAME%\n$pkgname\n" + echo -e "%VERSION%\n$pkgver-$pkgrel\n" + echo -e "%PATH%\n$dir\n" + + if [ -n "$arch" ]; then + echo "%ARCH%" + for i in ${arch[@]}; do echo $i; done + echo "" + fi + if [ -n "$depends" ]; then + echo "%DEPENDS%" + for i in ${depends[@]}; do + echo $i + done + echo "" + fi + if [ -n "$makedepends" ]; then + echo "%MAKEDEPENDS%" + for i in ${makedepends[@]}; do + echo $i + done + echo "" + fi + if [ -n "$conflicts" ]; then + echo "%CONFLICTS%" + for i in ${conflicts[@]}; do echo $i; done + echo "" + fi + if [ -n "$provides" ]; then + echo "%PROVIDES%" + for i in ${provides[@]}; do echo $i; done + echo "" + fi + return 0 +} + +find_pkgbuilds() { + if [ -f $1/PKGBUILD ]; then + parse $1 + return + fi + empty=1 + for dir in $1/*; do + if [ -d $dir ]; then + find_pkgbuilds $dir + unset empty + fi + done + if [ -n "$empty" ]; then + echo -e "%MISSING%\n$1\n" + fi +} + +if [ -z "$*" ]; then + exit 1 +fi + +for dir in "$@"; do + find_pkgbuilds $dir +done + +exit 0 diff --git a/cron-jobs/check_archlinux/setup.py b/cron-jobs/check_archlinux/setup.py new file mode 100644 index 0000000..b172752 --- /dev/null +++ b/cron-jobs/check_archlinux/setup.py @@ -0,0 +1,10 @@ +from distutils.core import setup, Extension + +alpm = Extension('alpm', + libraries = ['alpm'], + sources = ['alpm.c']) + +setup (name = 'Alpm', + version = '1.0', + description = 'Alpm bindings', + ext_modules = [alpm]) -- cgit v1.2.3-2-g168b From 7f0886c7e8935352007d6ba8acc6d9e28b55f2da Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sat, 30 Aug 2008 21:05:22 -0500 Subject: Fix an error message copy/paste error Signed-off-by: Aaron Griffin --- cron-jobs/ftpdir-cleanup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index ce91b62..65f0855 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -17,7 +17,7 @@ ctrl_c() { if [ -f "$LOCKFILE" ]; then owner="$(/usr/bin/stat -c %U $LOCKFILE)" - echo "error: source tarball generation is already in progress (started by $owner)" + echo "error: ftp cleanup is already in progress (started by $owner)" exit 1 fi -- cgit v1.2.3-2-g168b From e1ad729fd49c630eece0055ed996b382811a836c Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sun, 31 Aug 2008 01:07:26 -0400 Subject: Add sourceball generating cron script Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 81 ++++++++++++++++++++++++++++++++++++++++++++++ cron-jobs/sourceballs.skip | 1 + 2 files changed, 82 insertions(+) create mode 100755 cron-jobs/sourceballs create mode 100644 cron-jobs/sourceballs.skip (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs new file mode 100755 index 0000000..f599ff1 --- /dev/null +++ b/cron-jobs/sourceballs @@ -0,0 +1,81 @@ +#!/bin/bash + +ftpbase="/home/ftp" +srcbase="/home/aaron/public_html/sources/" +repos="core extra unstable testing" +arches="i686 x86_64" + +LOCKFILE="/tmp/.sourceball.lock" + +if [ ! -f /etc/makepkg.conf ]; then + echo "/etc/makepkg.conf not found! Aborting" + exit 1 +fi + +. /etc/makepkg.conf +cleanup () { + rm -f "$LOCKFILE" + exit 0 +} + +ctrl_c() { + cleanup +} + +if [ -f "$LOCKFILE" ]; then + owner="$(/usr/bin/stat -c %U $LOCKFILE)" + echo "error: source tarball generation is already in progress (started by $owner)" + exit 1 +fi + +trap cleanup 0 +trap ctrl_c 2 + +/bin/touch "$LOCKFILE" + +dirname="$(/bin/readlink -f $(/usr/bin/dirname $0))" + +getpkgname() { + local tmp + + tmp=${1##*/} + tmp=${tmp%$PKGEXT} + for a in $arches; do + tmp=${tmp%-$a} + done + echo ${tmp%-*-*} +} + +FAILED_PKGS="" + +for repo in $repos; do + for arch in $arches; do + export CARCH="$arch" + ftppath="$ftpbase/$repo/os/$arch" + cd $ftppath + for pkg in *$PKGEXT; do + pkgname=$(getpkgname $pkg) + srcpath="$srcbase/" + srcpkg="${pkg//-$arch$PKGEXT/$SRCEXT}" + + if grep $pkgname "$dirname/sourceballs.skip" >/dev/null 2>&1; then + echo ":: Skipping package '$pkgname'" + continue + fi + + if [ ! -f "$srcpath$srcpkg" ]; then + if ! $dirname/../misc-scripts/make-sourceball $pkgname $repo $arch; then + FAILED_PKGS="$FAILED_PKGS $pkg" + fi + fi + done + done +done + +if [ -n "$FAILED_PKGS" ]; then + echo "" + echo "The following packages failed:" + echo -e $FAILED_PKGS | sed "s| |\n\t|g" | sort -u +fi + +cleanup diff --git a/cron-jobs/sourceballs.skip b/cron-jobs/sourceballs.skip new file mode 100644 index 0000000..0b5a58e --- /dev/null +++ b/cron-jobs/sourceballs.skip @@ -0,0 +1 @@ +readline -- cgit v1.2.3-2-g168b From 766727fa10cbc2715baba93437fbb30b798f08c9 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sun, 31 Aug 2008 00:08:54 -0500 Subject: Add sourceball cron job at the proper path (whoops) Also remove readline from the skip-list Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 2 +- cron-jobs/sourceballs.skip | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index f599ff1..ee62d46 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -1,7 +1,7 @@ #!/bin/bash ftpbase="/home/ftp" -srcbase="/home/aaron/public_html/sources/" +srcbase="/home/ftp/sources" repos="core extra unstable testing" arches="i686 x86_64" diff --git a/cron-jobs/sourceballs.skip b/cron-jobs/sourceballs.skip index 0b5a58e..e69de29 100644 --- a/cron-jobs/sourceballs.skip +++ b/cron-jobs/sourceballs.skip @@ -1 +0,0 @@ -readline -- cgit v1.2.3-2-g168b From 6fc5ac8adc6bd66770351b1d786f5895d83b2f97 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 2 Sep 2008 00:01:54 -0500 Subject: Remove 'unstable' from sourceball creation Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index ee62d46..fc313e3 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -2,7 +2,7 @@ ftpbase="/home/ftp" srcbase="/home/ftp/sources" -repos="core extra unstable testing" +repos="core extra testing" arches="i686 x86_64" LOCKFILE="/tmp/.sourceball.lock" -- cgit v1.2.3-2-g168b From a3a43e54b6c673829a06b9bedcf8072c0358e0aa Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 2 Sep 2008 00:12:46 -0500 Subject: Remove the unstable repository Signed-off-by: Aaron Griffin --- cron-jobs/adjust-permissions | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index 7f7e4c8..88ec006 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -9,13 +9,13 @@ fi cd /home/ftp /bin/chown -R ftp:ftp-arch core/os/i686 /bin/chown -R ftp:ftp-arch core/os/x86_64 -/bin/chown -R ftp:ftp-extra {extra,unstable,testing}/os/i686 -/bin/chown -R ftp:ftp-extra {extra,unstable,testing}/os/x86_64 +/bin/chown -R ftp:ftp-extra {extra,testing}/os/i686 +/bin/chown -R ftp:ftp-extra {extra,testing}/os/x86_64 /bin/chown -R ftp:aur community /bin/chmod -R g+w /home/aur/unsupported{,-temp} -for d in core extra unstable testing community; do +for d in core extra testing community; do /bin/chmod -R g+w $d/os/i686 /bin/chmod -R g+w $d/os/x86_64 done -- cgit v1.2.3-2-g168b From 4bef49c9d906a5388027eb5cbe694b0b6b93e807 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 2 Sep 2008 00:13:40 -0500 Subject: Cron-job changes Signed-off-by: Aaron Griffin --- cron-jobs/repo-maint.crontab | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/repo-maint.crontab b/cron-jobs/repo-maint.crontab index dadd1fb..babcea1 100644 --- a/cron-jobs/repo-maint.crontab +++ b/cron-jobs/repo-maint.crontab @@ -8,5 +8,6 @@ MAILTO="root" #| | | | | commands # #################################################################### */5 * * * * /arch/cron-jobs/adjust-permissions -00 21 * * * /arch/cron-jobs/createFileList +00 21 * * * /arch/cron-jobs/createFileList >/dev/null 2>&1 +00 23 * * * /arch/cron-jobs/sourceballs 00 */3 * * * /arch/cron-jobs/ftpdir-cleanup | /arch/cron-jobs/devlist-mailer -- cgit v1.2.3-2-g168b From 86ec8b5c9dcd5b83cee55e410d42784c830efbb6 Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Sat, 27 Sep 2008 00:56:24 +0200 Subject: check_packages : improve the "skip klibc dep" hack. It is not possible to check the klibc dep on the PKGBUILD level, so I made a hack to skip it. This hack broke on klibc-jfflyAahxqaliwAofrf_fdf5upI because of the underscore. But we can simply use the \w regexp which matches any alpha-numeric char, including underscore. According to klibc developer, dashes are possible too. Also the length of this string is always 27 chars, so we can use that. Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/check_packages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index 2d2efbe..69b14c3 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -363,7 +363,7 @@ for name,pkg in repopkgs.iteritems(): # ugly hack to strip the weird kblic- deps for name,pkg in packages.iteritems(): - p = re.compile('klibc-[A-Za-z0-9]{20,}|klibc-\*') + p = re.compile('klibc-[\w\-]{27}|klibc-\*') pkg.deps = [dep for dep in pkg.deps if not p.match(dep)] pkg.makedeps = [dep for dep in pkg.makedeps if not p.match(dep)] -- cgit v1.2.3-2-g168b From bdc50e3ecd4bd0a1e02e66f331ea7084cb61882b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 10 Nov 2008 10:43:13 -0800 Subject: Remove chmod on /home/aur/* This one portion of the adjust-permissions script takes over a minute of wall clock time to run on gerolde when no other cron jobs are running. It takes three minutes when we hit the hour mark on the clock due to everything else going at the same time. A find on /home/aur/unsupported reveals this command will need to touch ~77000 files. Signed-off-by: Dan McGee Signed-off-by: Aaron Griffin --- cron-jobs/adjust-permissions | 2 -- 1 file changed, 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index 88ec006..232d48a 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -13,8 +13,6 @@ cd /home/ftp /bin/chown -R ftp:ftp-extra {extra,testing}/os/x86_64 /bin/chown -R ftp:aur community -/bin/chmod -R g+w /home/aur/unsupported{,-temp} - for d in core extra testing community; do /bin/chmod -R g+w $d/os/i686 /bin/chmod -R g+w $d/os/x86_64 -- cgit v1.2.3-2-g168b From 0dcc6b02d0ed7cf25e9493667030ded9525473ce Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 10 Nov 2008 10:45:29 -0800 Subject: cron-jobs: use renice to lower job priority Lower the job priority of our cron jobs so they don't interfere with other more important things on the server. None of these are very CPU intensive, but priority for I/O operations should go elsewhere. Signed-off-by: Dan McGee Signed-off-by: Aaron Griffin --- cron-jobs/adjust-permissions | 3 +++ cron-jobs/createFileLists | 3 +++ cron-jobs/ftpdir-cleanup | 3 +++ 3 files changed, 9 insertions(+) (limited to 'cron-jobs') diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index 232d48a..1493f4b 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -6,6 +6,9 @@ fi /bin/touch /tmp/.ftpmaint.lck +#adjust the nice level to run at a lower priority +/usr/bin/renice +10 -p $$ > /dev/null + cd /home/ftp /bin/chown -R ftp:ftp-arch core/os/i686 /bin/chown -R ftp:ftp-arch core/os/x86_64 diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists index d6cbd18..044b6d7 100755 --- a/cron-jobs/createFileLists +++ b/cron-jobs/createFileLists @@ -15,6 +15,9 @@ touch "/tmp/createFileList.lock" || exit 1 TMPDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1 CACHEDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1 +#adjust the nice level to run at a lower priority +/usr/bin/renice +10 -p $$ > /dev/null + getpkgname() { local tmp diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 65f0855..8b73956 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -26,6 +26,9 @@ trap ctrl_c 2 /bin/touch "$LOCKFILE" +#adjust the nice level to run at a lower priority +/usr/bin/renice +10 -p $$ > /dev/null + for repo in $repos; do for arch in $arches; do $(dirname $0)/../misc-scripts/ftpdir-cleanup $repo $arch -- cgit v1.2.3-2-g168b From 836152eed8893d6884796363419ce61ea7b7b417 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 21 Nov 2008 01:42:24 -0500 Subject: Remove unused cron-job 'genpkglist' Signed-off-by: Aaron Griffin --- cron-jobs/genpkglist | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100755 cron-jobs/genpkglist (limited to 'cron-jobs') diff --git a/cron-jobs/genpkglist b/cron-jobs/genpkglist deleted file mode 100755 index 0b5d76f..0000000 --- a/cron-jobs/genpkglist +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -if [ $# -ne 2 ]; then - echo "usage: $(basename $0) " - exit 1 -fi - -reponame=$1 -arch=$2 - -##### Arch specific stuff. TODO make this configurable ##### -ftppath="/home/ftp/$reponame/os/$arch/" -############################################################ - -if [ ! -d "$ftppath" ]; then - echo "FTP path '$ftppath' does not exist" - exit 1 -fi - -if [ ! -f /etc/makepkg.conf ]; then - echo "/etc/makepkg.conf not found! Aborting" - exit 1 -fi - -. /etc/makepkg.conf - -cd "$ftppath" -ls -1 *$PKGEXT -- cgit v1.2.3-2-g168b From eada2876aa5860edc17949154a505b8a431b1b83 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 21 Nov 2008 01:42:56 -0500 Subject: Move cron-jobs to /srv from /home Signed-off-by: Aaron Griffin --- cron-jobs/adjust-permissions | 4 ++-- cron-jobs/createFileLists | 2 +- cron-jobs/sourceballs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index 1493f4b..307c319 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -9,7 +9,7 @@ fi #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null -cd /home/ftp +cd /srv/ftp /bin/chown -R ftp:ftp-arch core/os/i686 /bin/chown -R ftp:ftp-arch core/os/x86_64 /bin/chown -R ftp:ftp-extra {extra,testing}/os/i686 @@ -20,6 +20,6 @@ for d in core extra testing community; do /bin/chmod -R g+w $d/os/i686 /bin/chmod -R g+w $d/os/x86_64 done -/bin/chmod 555 /home/ftp +/bin/chmod 555 /srv/ftp rm -f /tmp/.ftpmaint.lck diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists index 044b6d7..9ce108c 100755 --- a/cron-jobs/createFileLists +++ b/cron-jobs/createFileLists @@ -1,6 +1,6 @@ #!/bin/bash -reposdir=/home/ftp/ +reposdir=/srv/ftp/ #targetdir=/home/pierre/public_html/test-repo/ targetdir=$reposdir repos="core extra unstable testing community" diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index fc313e3..00ca5e3 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -1,7 +1,7 @@ #!/bin/bash -ftpbase="/home/ftp" -srcbase="/home/ftp/sources" +ftpbase="/srv/ftp" +srcbase="/srv/ftp/sources" repos="core extra testing" arches="i686 x86_64" -- cgit v1.2.3-2-g168b From 6b1f06f59fd13379841b1d3cdc5e51cbf8909e66 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 21 Nov 2008 15:22:46 -0500 Subject: Drop unstable from processing in scripts Signed-off-by: Dan McGee Signed-off-by: Aaron Griffin --- cron-jobs/createFileLists | 2 +- cron-jobs/ftpdir-cleanup | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists index 9ce108c..697c39d 100755 --- a/cron-jobs/createFileLists +++ b/cron-jobs/createFileLists @@ -3,7 +3,7 @@ reposdir=/srv/ftp/ #targetdir=/home/pierre/public_html/test-repo/ targetdir=$reposdir -repos="core extra unstable testing community" +repos="core extra testing community" arches="i686 x86_64" if [ -f "/tmp/createFileList.lock" ]; then diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 8b73956..1f18b17 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -1,7 +1,7 @@ #!/bin/bash #TODO add community -repos="core extra unstable testing" +repos="core extra testing" arches="i686 x86_64" LOCKFILE="/tmp/.ftpdircleanup.lock" -- cgit v1.2.3-2-g168b From 81efd3d8cd39d07930b18dcd8f904153500bfbf3 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 29 Dec 2008 13:30:00 -0800 Subject: Allow devlist-mailer to send to alternate addresses Signed-off-by: Aaron Griffin --- cron-jobs/devlist-mailer | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer index b24d530..671cbbb 100755 --- a/cron-jobs/devlist-mailer +++ b/cron-jobs/devlist-mailer @@ -7,8 +7,12 @@ LIST="arch-dev-public@archlinux.org" FROM="repomaint@archlinux.org" SUBJECT="Repository Maintenance $(date +"%d-%m-%Y %H:%M")" -if [ $# -eq 1 ]; then - SUBJECT="$1" +if [ $# -ge 1 ]; then + SUBJECT="$1 $(date +"%d-%m-%Y %H:%M")" +fi + +if [ $# -ge 2 ]; then + LIST="$2" fi stdin="$(cat)" -- cgit v1.2.3-2-g168b From d39b68ede8567fb2ee15f45995ddab8f4710e9dc Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Sat, 3 Jan 2009 13:22:57 +0100 Subject: check_packages : add --arch option. The parsing script didn't set CARCH previously, and the flashplugin PKGBUILD exited in this case. First override the exit function to prevent the whole script to exit, and add a --arch option to be able to set CARCH correctly. To be used like this : For core and extra : ./check_packages.py --abs-tree=/home/abs/rsync/i686 --repos=core,extra --arch=i686 ./check_packages.py --abs-tree=/home/abs/rsync/x86_64 --repos=core,extra --arch=x86_64 For community : ./check_packages.py --abs-tree=/home/abs/rsync/i686 --repos=community --arch=i686 ./check_packages.py --abs-tree=/home/abs/rsync/x86_64 --repos=community --arch=x86_64 Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/check_packages.py | 23 ++++++++++++++--------- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 9 ++++++++- 2 files changed, 22 insertions(+), 10 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index 69b14c3..f0fd4a5 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -49,11 +49,11 @@ class Depend: self.version = version self.mod = mod -def parse_pkgbuilds(repos): +def parse_pkgbuilds(repos,arch): oldcwd = os.getcwd() os.chdir(absroot) for repo in repos: - data = commands.getoutput(oldcwd + '/parse_pkgbuilds.sh ' + repo) + data = commands.getoutput(oldcwd + '/parse_pkgbuilds.sh ' + arch + ' ' + repo) parse_data(repo,data) os.chdir(oldcwd) @@ -292,24 +292,27 @@ def print_usage(): print "Usage: ./check_packages.py [OPTION]" print "" print "Options:" - print " --abs-tree= Check specified tree (default : /var/abs)" - print " --repos= Check specified repos (default : core,extra)" + print " --abs-tree= Check the specified tree (default : /var/abs)" + print " --repos= Check the specified repos (default : core,extra)" + print " --arch= Check the specified arch (default : i686)" print " -h, --help Show this help and exit" print "" print "Examples:" print "\n Check core and extra in existing abs tree:" - print " ./check_packages.py --abs-tree=/var/abs --repos=core,extra" + print " ./check_packages.py --abs-tree=/var/abs --repos=core,extra --arch=i686" print "\n Check community:" - print " ./check_packages.py --abs-tree=/var/abs --repos=community" + print " ./check_packages.py --abs-tree=/var/abs --repos=community --arch=i686" print "" ## Default path to the abs root directory absroot = "/var/abs" ## Default list of repos to check repos = ['core', 'extra'] +## Default arch +arch = "i686" try: - opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos="]) + opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos=", "arch="]) except getopt.GetoptError: print_usage() sys.exit() @@ -319,6 +322,8 @@ if opts != []: absroot = a elif o in ("--repos"): repos = a.split(",") + elif o in ("--arch"): + arch = a else: print_usage() sys.exit() @@ -339,11 +344,11 @@ loadrepos = set([]) for repo in repos: loadrepos = loadrepos | set(get_repo_hierarchy(repo)) -print_heading("Integrity Check") +print_heading("Integrity Check " + arch + " of " + ",".join(repos)) print "\nPerforming integrity checks..." print "==> parsing pkgbuilds" -parse_pkgbuilds(loadrepos) +parse_pkgbuilds(loadrepos,arch) repopkgs = {} for name,pkg in packages.iteritems(): diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index d4205ae..47aec89 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -1,5 +1,10 @@ #!/bin/bash +# Usage : parse_pkgbuilds.sh arch +# Example : parse_pkgbuilds.sh i686 /var/abs/core /var/abs/extra + +exit() { return; } + parse() { unset pkgname pkgver pkgrel unset depends makedepends conflicts provides @@ -67,10 +72,12 @@ find_pkgbuilds() { fi } -if [ -z "$*" ]; then +if [ -z "$1" -o -z "$2" ]; then exit 1 fi +CARCH=$1 +shift for dir in "$@"; do find_pkgbuilds $dir done -- cgit v1.2.3-2-g168b From 5f64e9993a494040b0280ed0f1daa38f68f9c46b Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 14 Jan 2009 13:31:11 -0800 Subject: Remove HH:MM from devlist-mailer subject lines This was extraneous info. Not needed as higher granularity can be gleaned from the mail's date Signed-off-by: Aaron Griffin --- cron-jobs/devlist-mailer | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer index 671cbbb..ca2e46b 100755 --- a/cron-jobs/devlist-mailer +++ b/cron-jobs/devlist-mailer @@ -6,9 +6,9 @@ LIST="arch-dev-public@archlinux.org" #LIST="aaronmgriffin@gmail.com" FROM="repomaint@archlinux.org" -SUBJECT="Repository Maintenance $(date +"%d-%m-%Y %H:%M")" +SUBJECT="Repository Maintenance $(date +"%d-%m-%Y")" if [ $# -ge 1 ]; then - SUBJECT="$1 $(date +"%d-%m-%Y %H:%M")" + SUBJECT="$1 $(date +"%d-%m-%Y")" fi if [ $# -ge 2 ]; then -- cgit v1.2.3-2-g168b From d9df504ddd648171eeff73d63afe196e153459a2 Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Sun, 1 Feb 2009 20:59:10 +0100 Subject: check_packages: fix stupid assumption on script location. The parse_pkgbuilds.sh script was assumed to be in the current working directory, which is quite stupid since check_packages.py can be called from anywhere. Now it only assumes that check_packages.py and parse_pkgbuilds.sh are in the same directory. This should fix the empty integrity checks on arch-dev :) Signed-off-by: Xavier Chantry Signed-off-by: Dan McGee --- cron-jobs/check_archlinux/check_packages.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index f0fd4a5..0392368 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -50,12 +50,10 @@ class Depend: self.mod = mod def parse_pkgbuilds(repos,arch): - oldcwd = os.getcwd() - os.chdir(absroot) for repo in repos: - data = commands.getoutput(oldcwd + '/parse_pkgbuilds.sh ' + arch + ' ' + repo) + data = commands.getoutput(os.path.dirname(sys.argv[0]) + '/parse_pkgbuilds.sh ' + + arch + ' ' + absroot + '/' + repo) parse_data(repo,data) - os.chdir(oldcwd) def parse_data(repo,data): attrname = None -- cgit v1.2.3-2-g168b From 9df5198acf0088ce8818bee9b3c9fbf44a26628d Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 4 Feb 2009 16:15:31 -0500 Subject: Remove 'skipping' output. Not needed Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 00ca5e3..03b8209 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -46,6 +46,7 @@ getpkgname() { echo ${tmp%-*-*} } + FAILED_PKGS="" for repo in $repos; do @@ -59,7 +60,6 @@ for repo in $repos; do srcpkg="${pkg//-$arch$PKGEXT/$SRCEXT}" if grep $pkgname "$dirname/sourceballs.skip" >/dev/null 2>&1; then - echo ":: Skipping package '$pkgname'" continue fi -- cgit v1.2.3-2-g168b From a5eb44ed1bbdf2eabd26cc02932898719375c230 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 9 Feb 2009 15:36:04 -0600 Subject: Write out failed packages to a txt file Located at /srv/ftp/sources/failed.txt Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 03b8209..9509715 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -73,9 +73,11 @@ for repo in $repos; do done if [ -n "$FAILED_PKGS" ]; then + echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u > "$srcbase/failed.txt" + echo "" echo "The following packages failed:" - echo -e $FAILED_PKGS | sed "s| |\n\t|g" | sort -u + cat "$srcbase/failed.txt" fi cleanup -- cgit v1.2.3-2-g168b From 75de4d131ed424bfb7c349a6b55cedf9705879bd Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 11 Feb 2009 11:46:02 -0500 Subject: Output stderr to an error file for each run Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 9509715..31ac1af 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -49,6 +49,8 @@ getpkgname() { FAILED_PKGS="" +echo "Errors occured during run:" > "$srcbase/errors.txt" + for repo in $repos; do for arch in $arches; do export CARCH="$arch" @@ -64,7 +66,8 @@ for repo in $repos; do fi if [ ! -f "$srcpath$srcpkg" ]; then - if ! $dirname/../misc-scripts/make-sourceball $pkgname $repo $arch; then + if ! $dirname/../misc-scripts/make-sourceball \ + $pkgname $repo $arch 2>>"$srcbase/errors.txt"; then FAILED_PKGS="$FAILED_PKGS $pkg" fi fi @@ -73,11 +76,8 @@ for repo in $repos; do done if [ -n "$FAILED_PKGS" ]; then - echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u > "$srcbase/failed.txt" - - echo "" - echo "The following packages failed:" - cat "$srcbase/failed.txt" + echo "The following packages failed:" > "$srcbase/failed.txt" + echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u >> "$srcbase/failed.txt" fi cleanup -- cgit v1.2.3-2-g168b From 3c8878d337d59f46d37446ddde2b40af69d36330 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 13 Feb 2009 12:14:41 -0800 Subject: Allow skipping of sourceball license check Also add a whitelist of files to skip (empty right now) Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 10 +++++++++- cron-jobs/sourceballs.force | 0 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 cron-jobs/sourceballs.force (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 31ac1af..b709d14 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -61,12 +61,20 @@ for repo in $repos; do srcpath="$srcbase/" srcpkg="${pkg//-$arch$PKGEXT/$SRCEXT}" + #Don't do anything for package in this 'blacklist' if grep $pkgname "$dirname/sourceballs.skip" >/dev/null 2>&1; then continue fi + #Use this file to 'whitelist' or force building some sourceballs, + # skipping the license check + force="" + if grep $pkgname "$dirname/sourceballs.force" >/dev/null 2>&1; then + force="-f" + fi + if [ ! -f "$srcpath$srcpkg" ]; then - if ! $dirname/../misc-scripts/make-sourceball \ + if ! $dirname/../misc-scripts/make-sourceball $force \ $pkgname $repo $arch 2>>"$srcbase/errors.txt"; then FAILED_PKGS="$FAILED_PKGS $pkg" fi diff --git a/cron-jobs/sourceballs.force b/cron-jobs/sourceballs.force new file mode 100644 index 0000000..e69de29 -- cgit v1.2.3-2-g168b From 83093650bda19395e5611c498752c5f2647157f8 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 13 Feb 2009 12:19:21 -0800 Subject: Fix pkgname parsing for files without ARCH Split the srcpkg parsing into two separate steps, one to swap the extensions, and another to remove the architecture Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index b709d14..07bc7aa 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -59,7 +59,8 @@ for repo in $repos; do for pkg in *$PKGEXT; do pkgname=$(getpkgname $pkg) srcpath="$srcbase/" - srcpkg="${pkg//-$arch$PKGEXT/$SRCEXT}" + srcpkg="${pkg//$PKGEXT/$SRCEXT}" + srcpkg="${srcpkg//-$arch/}" #Don't do anything for package in this 'blacklist' if grep $pkgname "$dirname/sourceballs.skip" >/dev/null 2>&1; then -- cgit v1.2.3-2-g168b From 918e009a6468587cc442c1ed62a5f0264bd28d32 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 13 Feb 2009 14:02:49 -0800 Subject: Keep the last failed.txt too for comparison Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 1 + 1 file changed, 1 insertion(+) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 07bc7aa..470dd7e 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -85,6 +85,7 @@ for repo in $repos; do done if [ -n "$FAILED_PKGS" ]; then + [ -e "$srcbase/failed.txt" ] && /bin/mv "$srcbase/failed.txt" "$srcbase/failed.txt.old" echo "The following packages failed:" > "$srcbase/failed.txt" echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u >> "$srcbase/failed.txt" fi -- cgit v1.2.3-2-g168b From 8d32b23c487fb2cd1a27ff350547ca1a2cdb5f68 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 16 Feb 2009 19:52:36 -0800 Subject: sourceballs: copy previous errors.txt output file Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 1 + 1 file changed, 1 insertion(+) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 470dd7e..d36da4b 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -49,6 +49,7 @@ getpkgname() { FAILED_PKGS="" +[ -e "$srcbase/errors.txt" ] && /bin/mv "$srcbase/errors.txt" "$srcbase/errors.txt.old" echo "Errors occured during run:" > "$srcbase/errors.txt" for repo in $repos; do -- cgit v1.2.3-2-g168b From db00ce3c343cf6eea9cb512f7bba9ea7904526b8 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sat, 21 Feb 2009 13:43:23 -0800 Subject: Add a script to simplify the integ check call Signed-off-by: Aaron Griffin --- cron-jobs/integrity-check | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100755 cron-jobs/integrity-check (limited to 'cron-jobs') diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check new file mode 100755 index 0000000..c3b016c --- /dev/null +++ b/cron-jobs/integrity-check @@ -0,0 +1,14 @@ +#!/bin/bash + +basedir="$(dirname $0)" + +if [ $# -ne 3 ]; then + echo "usage: $(basename $0) [,,...] " >2 + exit 1 +fi + +$basedir/cron-jobs/check_archlinux/check_packages.py \ + --repos="$1" \ + --abs-tree="/srv/abs/rsync/$2" --arch="$1" |\ + $basedir/cron-jobs/devlist-mailer \ + "Integrity Check $2: $1" "$3" -- cgit v1.2.3-2-g168b From 110f40248d3e365a98f82d30e2256da35689fbf7 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sat, 21 Feb 2009 13:52:19 -0800 Subject: Remove the repo-maint crontab Our cron currently doesn't handle stand-alone crontabs too well, and we're managing all these in root's crontab anyway. Meh... deleted Signed-off-by: Aaron Griffin --- cron-jobs/repo-maint.crontab | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 cron-jobs/repo-maint.crontab (limited to 'cron-jobs') diff --git a/cron-jobs/repo-maint.crontab b/cron-jobs/repo-maint.crontab deleted file mode 100644 index babcea1..0000000 --- a/cron-jobs/repo-maint.crontab +++ /dev/null @@ -1,13 +0,0 @@ -MAILTO="root" -#################################################################### -#minute (0-59), # -#| hour (0-23), # -#| | day of the month (1-31), # -#| | | month of the year (1-12), # -#| | | | day of the week (0-6 with 0=Sunday)# -#| | | | | commands # -#################################################################### -*/5 * * * * /arch/cron-jobs/adjust-permissions -00 21 * * * /arch/cron-jobs/createFileList >/dev/null 2>&1 -00 23 * * * /arch/cron-jobs/sourceballs -00 */3 * * * /arch/cron-jobs/ftpdir-cleanup | /arch/cron-jobs/devlist-mailer -- cgit v1.2.3-2-g168b From 0d3c5671c27f0b5848bb72df9af2270f72e7c988 Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Sat, 21 Feb 2009 17:16:17 -0800 Subject: integrity-check: fix a few errors I think it will work better now :) Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/integrity-check | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index c3b016c..0b59064 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -3,12 +3,11 @@ basedir="$(dirname $0)" if [ $# -ne 3 ]; then - echo "usage: $(basename $0) [,,...] " >2 + echo "usage: $(basename $0) [,,...] " >&2 exit 1 fi -$basedir/cron-jobs/check_archlinux/check_packages.py \ +$basedir/check_archlinux/check_packages.py \ --repos="$1" \ - --abs-tree="/srv/abs/rsync/$2" --arch="$1" |\ - $basedir/cron-jobs/devlist-mailer \ - "Integrity Check $2: $1" "$3" + --abs-tree="/srv/abs/rsync/$2" --arch="$2" |\ + $basedir/devlist-mailer "Integrity Check $2: $1" "$3" -- cgit v1.2.3-2-g168b From 2de2859cc0fd7f9db26630dffec25fccb88f3434 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 23 Feb 2009 13:50:34 -0800 Subject: Ensure grep matches only full pkgnames Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index d36da4b..b4085f2 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -64,14 +64,14 @@ for repo in $repos; do srcpkg="${srcpkg//-$arch/}" #Don't do anything for package in this 'blacklist' - if grep $pkgname "$dirname/sourceballs.skip" >/dev/null 2>&1; then + if grep "^$pkgname\$" "$dirname/sourceballs.skip" >/dev/null 2>&1; then continue fi #Use this file to 'whitelist' or force building some sourceballs, # skipping the license check force="" - if grep $pkgname "$dirname/sourceballs.force" >/dev/null 2>&1; then + if grep "^$pkgname\$" "$dirname/sourceballs.force" >/dev/null 2>&1; then force="-f" fi -- cgit v1.2.3-2-g168b From dbca654de502b1a6fb925544256dcfc3b54cc623 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Charette?= Date: Wed, 6 May 2009 12:31:26 -0700 Subject: adjust_permissions: Add 'any' architecture Signed-off-by: Aaron Griffin --- cron-jobs/adjust-permissions | 3 +++ 1 file changed, 3 insertions(+) (limited to 'cron-jobs') diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index 307c319..947c021 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -10,13 +10,16 @@ fi /usr/bin/renice +10 -p $$ > /dev/null cd /srv/ftp +/bin/chown -R ftp:ftp-arch core/os/any /bin/chown -R ftp:ftp-arch core/os/i686 /bin/chown -R ftp:ftp-arch core/os/x86_64 +/bin/chown -R ftp:ftp-extra {extra,testing}/os/any /bin/chown -R ftp:ftp-extra {extra,testing}/os/i686 /bin/chown -R ftp:ftp-extra {extra,testing}/os/x86_64 /bin/chown -R ftp:aur community for d in core extra testing community; do + /bin/chmod -R g+w $d/os/any /bin/chmod -R g+w $d/os/i686 /bin/chmod -R g+w $d/os/x86_64 done -- cgit v1.2.3-2-g168b From 8e9a5b387a76f31796837e1000188d1152bd4b00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Charette?= Date: Wed, 6 May 2009 12:32:43 -0700 Subject: check_packages.py: Add 'any' arch support Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/check_packages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index 0392368..9bdd9c1 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -194,7 +194,7 @@ def get_repo_hierarchy(repo): return ['core','extra','community'] def verify_archs(name,archs): - valid_archs = ['i686', 'x86_64'] + valid_archs = ['any', 'i686', 'x86_64'] invalid_archs = [] for arch in archs: if arch not in valid_archs: @@ -292,7 +292,7 @@ def print_usage(): print "Options:" print " --abs-tree= Check the specified tree (default : /var/abs)" print " --repos= Check the specified repos (default : core,extra)" - print " --arch= Check the specified arch (default : i686)" + print " --arch= Check the specified arch (default : i686)" print " -h, --help Show this help and exit" print "" print "Examples:" -- cgit v1.2.3-2-g168b From f515029a694a11065f95a79bd83b325bf04562bb Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 8 May 2009 13:21:39 -0700 Subject: check_archlinux: Skip searching CVS and .svn dirs Prevents some errors in the community scripts due to scanning of CVS dirs. Also skipping .svn dirs for the future (dotglob may be set, in which case we'd scan .svn dirs as well) Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index 47aec89..7dcfbbb 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -56,6 +56,12 @@ parse() { } find_pkgbuilds() { + #Skip over some dirs + local d="$(basename $1)" + if [ "$d" = "CVS" -o "$d" = ".svn" ]; then + return + fi + if [ -f $1/PKGBUILD ]; then parse $1 return -- cgit v1.2.3-2-g168b From c076b93fafbf47a85bd063c626621552f364a59d Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Wed, 13 May 2009 11:06:06 +0200 Subject: use pkgext defined in makepkg.conf Signed-off-by: Pierre Schmitz Signed-off-by: Aaron Griffin --- cron-jobs/createFileLists | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists index 697c39d..6d03260 100755 --- a/cron-jobs/createFileLists +++ b/cron-jobs/createFileLists @@ -1,11 +1,17 @@ #!/bin/bash reposdir=/srv/ftp/ -#targetdir=/home/pierre/public_html/test-repo/ targetdir=$reposdir repos="core extra testing community" arches="i686 x86_64" +if [ -f /etc/makepkg.conf ]; then + . /etc/makepkg.conf +else + echo "Error: /etc/makepkg.conf does not exist." + exit 1 +fi + if [ -f "/tmp/createFileList.lock" ]; then echo "Error: createFileList allready in progress." exit 1 @@ -22,7 +28,7 @@ getpkgname() { local tmp tmp=${1##*/} - tmp=${tmp%.pkg.tar.gz} + tmp=${tmp%$PKGEXT} tmp=${tmp%-i686} tmp=${tmp%-x86_64} tmp=${tmp%-any} @@ -31,7 +37,7 @@ getpkgname() { cd $reposdir for repo in $repos; do - REPO_DB_FILE=${repo}.files.tar.gz + REPO_DB_FILE=${repo}.files.tar.${DB_COMPRESSION} for arch in $arches; do repodir=${repo}/os/${arch}/ cached="no" @@ -44,7 +50,7 @@ for repo in $repos; do fi # create file lists - for pkg in $repodir*.pkg.tar.gz; do + for pkg in $repodir*${PKGEXT}; do basename=$(basename $pkg) pkgname=$(getpkgname $basename) tmppkgdir=${TMPDIR}/${repodir}${pkgname}/ @@ -69,7 +75,7 @@ for repo in $repos; do cd ${TMPDIR}/${repodir} [ -f "${pkgdir}${REPO_DB_FILE}.old" ] && rm "${pkgdir}${REPO_DB_FILE}.old" [ -f "${pkgdir}${REPO_DB_FILE}" ] && mv "${pkgdir}${REPO_DB_FILE}" "${pkgdir}${REPO_DB_FILE}.old" - bsdtar --exclude=*.tar.gz -czf ${pkgdir}${REPO_DB_FILE} * + bsdtar --exclude=*.tar.${DB_COMPRESSION} -cf ${pkgdir}${REPO_DB_FILE} * fi cd $reposdir -- cgit v1.2.3-2-g168b From f1b7557d23c601a86a03cd5eee4feb4072a3b8b2 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 2 Jun 2009 19:31:47 +0200 Subject: compute compression option from file extension bsdtar does not detect the compression type from the file extension. So we need to set the right option on our own. Signed-off-by: Aaron Griffin --- cron-jobs/createFileLists | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists index 6d03260..10520af 100755 --- a/cron-jobs/createFileLists +++ b/cron-jobs/createFileLists @@ -35,6 +35,13 @@ getpkgname() { echo $tmp } +case "${DB_COMPRESSION}" in + gz) TAR_OPT="z" ;; + bz2) TAR_OPT="j" ;; + xz) TAR_OPT="J" ;; + *) echo "${DB_COMPRESSION} is not a valid archive compression" && exit 1 ;; +esac + cd $reposdir for repo in $repos; do REPO_DB_FILE=${repo}.files.tar.${DB_COMPRESSION} @@ -75,7 +82,7 @@ for repo in $repos; do cd ${TMPDIR}/${repodir} [ -f "${pkgdir}${REPO_DB_FILE}.old" ] && rm "${pkgdir}${REPO_DB_FILE}.old" [ -f "${pkgdir}${REPO_DB_FILE}" ] && mv "${pkgdir}${REPO_DB_FILE}" "${pkgdir}${REPO_DB_FILE}.old" - bsdtar --exclude=*.tar.${DB_COMPRESSION} -cf ${pkgdir}${REPO_DB_FILE} * + bsdtar --exclude=*.tar.${DB_COMPRESSION} -c${TAR_OPT}f ${pkgdir}${REPO_DB_FILE} * fi cd $reposdir -- cgit v1.2.3-2-g168b From 66d12ed4b83d623cfbda06242982f73e50a78125 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 17 Jul 2009 12:32:08 -0700 Subject: Fix adjust-permissions for gerolde and sigurd Signed-off-by: Aaron Griffin --- cron-jobs/adjust-permissions | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index 947c021..6670a6f 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -10,19 +10,28 @@ fi /usr/bin/renice +10 -p $$ > /dev/null cd /srv/ftp -/bin/chown -R ftp:ftp-arch core/os/any -/bin/chown -R ftp:ftp-arch core/os/i686 -/bin/chown -R ftp:ftp-arch core/os/x86_64 -/bin/chown -R ftp:ftp-extra {extra,testing}/os/any -/bin/chown -R ftp:ftp-extra {extra,testing}/os/i686 -/bin/chown -R ftp:ftp-extra {extra,testing}/os/x86_64 -/bin/chown -R ftp:aur community +if [ -d "core" ]; then + #This is unique to gerolde (main arch server) + /bin/chown -R ftp:ftp-arch core/os/any + /bin/chown -R ftp:ftp-arch core/os/i686 + /bin/chown -R ftp:ftp-arch core/os/x86_64 + /bin/chown -R ftp:ftp-extra {extra,testing}/os/any + /bin/chown -R ftp:ftp-extra {extra,testing}/os/i686 + /bin/chown -R ftp:ftp-extra {extra,testing}/os/x86_64 + for d in core extra testing; do + /bin/chmod -R g+w $d/os/any + /bin/chmod -R g+w $d/os/i686 + /bin/chmod -R g+w $d/os/x86_64 + done +else + /bin/chown -R root:tusers {community,community-testing}/os/any + /bin/chown -R root:tusers {community,community-testing}/os/i686 + /bin/chown -R root:tusers {community,community-testing}/os/x86_64 + /bin/chmod -R g+w {community,community-testing}/os/any + /bin/chmod -R g+w {community,community-testing}/os/i686 + /bin/chmod -R g+w {community,community-testing}/os/x86_64 +fi -for d in core extra testing community; do - /bin/chmod -R g+w $d/os/any - /bin/chmod -R g+w $d/os/i686 - /bin/chmod -R g+w $d/os/x86_64 -done /bin/chmod 555 /srv/ftp rm -f /tmp/.ftpmaint.lck -- cgit v1.2.3-2-g168b From 0667909b7a0a72cde7db32f7664daef7c402cd91 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 17 Jul 2009 12:33:16 -0700 Subject: Add community and 'any' arch to ftp cleanup Signed-off-by: Aaron Griffin --- cron-jobs/ftpdir-cleanup | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 1f18b17..6c295c9 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -1,8 +1,7 @@ #!/bin/bash -#TODO add community -repos="core extra testing" -arches="i686 x86_64" +repos="core extra testing community" +arches="any i686 x86_64" LOCKFILE="/tmp/.ftpdircleanup.lock" -- cgit v1.2.3-2-g168b From b354593994a96901f4b86a0fb734edb5cb1b2348 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 17 Jul 2009 12:35:35 -0700 Subject: Add community and 'any' arch to sourceballs Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index b4085f2..625d90d 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -2,8 +2,8 @@ ftpbase="/srv/ftp" srcbase="/srv/ftp/sources" -repos="core extra testing" -arches="i686 x86_64" +repos="core extra testing community" +arches="any i686 x86_64" LOCKFILE="/tmp/.sourceball.lock" -- cgit v1.2.3-2-g168b From 4b4a648c3ba4612d82ed1c6c23eb81fcef97fd14 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 17 Jul 2009 12:35:49 -0700 Subject: sourceballs: skip nonexistant FTP paths This is useful when running on separate machines that don't have all repos Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 625d90d..d07c482 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -56,6 +56,10 @@ for repo in $repos; do for arch in $arches; do export CARCH="$arch" ftppath="$ftpbase/$repo/os/$arch" + if [ ! -d "$ftppath" ]; then + echo "FTP path does not exist: $ftppath" >2 + continue + fi cd $ftppath for pkg in *$PKGEXT; do pkgname=$(getpkgname $pkg) -- cgit v1.2.3-2-g168b From 3bcb9d8c78857cb40110616c3fe08259c577fdca Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 17 Jul 2009 12:51:58 -0700 Subject: ftpdir-cleanup: Add configurable destdir Signed-off-by: Aaron Griffin --- cron-jobs/ftpdir-cleanup | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 6c295c9..77a0843 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -28,9 +28,12 @@ trap ctrl_c 2 #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null +#Get our destination dir +. $(dirname $0)/../config + for repo in $repos; do for arch in $arches; do - $(dirname $0)/../misc-scripts/ftpdir-cleanup $repo $arch + $(dirname $0)/../misc-scripts/ftpdir-cleanup $repo $arch $CLEANUP_DESTDIR done done -- cgit v1.2.3-2-g168b From 82cfca1d3d3e9f432678d7e687c6ca91731410c1 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 20 Jul 2009 19:02:57 -0400 Subject: ftpdir-cleanup: Don't scan 'any' dirs These are handled automatically when scanning other architecutres Signed-off-by: Aaron Griffin --- cron-jobs/ftpdir-cleanup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 77a0843..dfa968e 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -1,7 +1,7 @@ #!/bin/bash repos="core extra testing community" -arches="any i686 x86_64" +arches="i686 x86_64" LOCKFILE="/tmp/.ftpdircleanup.lock" -- cgit v1.2.3-2-g168b From 820452a93f68c05ac1083d56d9e560903d8ae1cf Mon Sep 17 00:00:00 2001 From: Francois Charette Date: Tue, 21 Jul 2009 11:38:09 +0200 Subject: Refactor ftpdir-cleanup to handle all arches This may look like a rather large patch, but the changes are mostly a reshuffling of the code to loop over all arches first, and then handle the arch-indep packages. The cronjob has been changed accordingly. Added new category DELETESYMLINKS, which are deleted instead of being moved to package-cleanup. I have also fixed the arch-specific issue with the ftppath, using parameters in config instead. Signed-off-by: Aaron Griffin --- cron-jobs/ftpdir-cleanup | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index dfa968e..38c7fcd 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -1,13 +1,12 @@ #!/bin/bash repos="core extra testing community" -arches="i686 x86_64" LOCKFILE="/tmp/.ftpdircleanup.lock" cleanup () { rm -f "$LOCKFILE" - exit 0 + exit 0 } ctrl_c() { @@ -32,9 +31,7 @@ trap ctrl_c 2 . $(dirname $0)/../config for repo in $repos; do - for arch in $arches; do - $(dirname $0)/../misc-scripts/ftpdir-cleanup $repo $arch $CLEANUP_DESTDIR - done + $(dirname $0)/../misc-scripts/ftpdir-cleanup $repo $CLEANUP_DESTDIR done cleanup -- cgit v1.2.3-2-g168b From 6ed6f4e6a482e913b01f5dc6d0b52575614742a7 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 22 Jul 2009 20:55:01 -0700 Subject: Use the top level config file for cron-jobs Source our config file, and use ARCHES from it Signed-off-by: Aaron Griffin --- cron-jobs/createFileLists | 5 +++-- cron-jobs/sourceballs | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists index 10520af..61cbc21 100755 --- a/cron-jobs/createFileLists +++ b/cron-jobs/createFileLists @@ -3,7 +3,8 @@ reposdir=/srv/ftp/ targetdir=$reposdir repos="core extra testing community" -arches="i686 x86_64" + +. $(dirname $0)/../config if [ -f /etc/makepkg.conf ]; then . /etc/makepkg.conf @@ -45,7 +46,7 @@ esac cd $reposdir for repo in $repos; do REPO_DB_FILE=${repo}.files.tar.${DB_COMPRESSION} - for arch in $arches; do + for arch in ${ARCHES[@]}; do repodir=${repo}/os/${arch}/ cached="no" diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index d07c482..27976ca 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -3,7 +3,8 @@ ftpbase="/srv/ftp" srcbase="/srv/ftp/sources" repos="core extra testing community" -arches="any i686 x86_64" + +. $(dirname $0)/../config LOCKFILE="/tmp/.sourceball.lock" @@ -40,7 +41,7 @@ getpkgname() { tmp=${1##*/} tmp=${tmp%$PKGEXT} - for a in $arches; do + for a in ${ARCHES[@]}; do tmp=${tmp%-$a} done echo ${tmp%-*-*} @@ -53,7 +54,7 @@ FAILED_PKGS="" echo "Errors occured during run:" > "$srcbase/errors.txt" for repo in $repos; do - for arch in $arches; do + for arch in ${ARCHES[@]}; do export CARCH="$arch" ftppath="$ftpbase/$repo/os/$arch" if [ ! -d "$ftppath" ]; then -- cgit v1.2.3-2-g168b From fcd15f215b7cb09e74e565799dd8b73748129aae Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 3 Aug 2009 12:29:11 -0400 Subject: Remove community from stock ftpdir-cleanup community repo is cleaned up on a different machine Signed-off-by: Aaron Griffin --- cron-jobs/ftpdir-cleanup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 38c7fcd..1a9fd98 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -1,6 +1,6 @@ #!/bin/bash -repos="core extra testing community" +repos="core extra testing" LOCKFILE="/tmp/.ftpdircleanup.lock" -- cgit v1.2.3-2-g168b From 1d94c827c8e9efe6e56bc15f993a1ce3764a675a Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 16 Aug 2009 09:04:21 -0500 Subject: Add DB_COMPRESSION var to all scripts that need it Signed-off-by: Dan McGee --- cron-jobs/createFileLists | 1 + 1 file changed, 1 insertion(+) (limited to 'cron-jobs') diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists index 61cbc21..2e42e56 100755 --- a/cron-jobs/createFileLists +++ b/cron-jobs/createFileLists @@ -3,6 +3,7 @@ reposdir=/srv/ftp/ targetdir=$reposdir repos="core extra testing community" +DB_COMPRESSION="gz" . $(dirname $0)/../config -- cgit v1.2.3-2-g168b From 6666d28f019b1b7ea38f9bf1f4371a87d42fb927 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 18 Aug 2009 16:01:54 -0700 Subject: More makepkg.conf removal - prefer toplevel config Signed-off-by: Aaron Griffin --- cron-jobs/createFileLists | 7 ------- cron-jobs/sourceballs | 6 ------ 2 files changed, 13 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists index 2e42e56..610f0e1 100755 --- a/cron-jobs/createFileLists +++ b/cron-jobs/createFileLists @@ -7,13 +7,6 @@ DB_COMPRESSION="gz" . $(dirname $0)/../config -if [ -f /etc/makepkg.conf ]; then - . /etc/makepkg.conf -else - echo "Error: /etc/makepkg.conf does not exist." - exit 1 -fi - if [ -f "/tmp/createFileList.lock" ]; then echo "Error: createFileList allready in progress." exit 1 diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 27976ca..a34ae2d 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -8,12 +8,6 @@ repos="core extra testing community" LOCKFILE="/tmp/.sourceball.lock" -if [ ! -f /etc/makepkg.conf ]; then - echo "/etc/makepkg.conf not found! Aborting" - exit 1 -fi - -. /etc/makepkg.conf cleanup () { rm -f "$LOCKFILE" exit 0 -- cgit v1.2.3-2-g168b From 49fb4d75ea83dca2ed4653e8454c478c1fd0555a Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 18 Aug 2009 16:11:03 -0700 Subject: More removal of DB_COMPRESSION from cron jobs Remove dependance on makepkg.conf Signed-off-by: Aaron Griffin --- cron-jobs/createFileLists | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists index 610f0e1..4ad4c22 100755 --- a/cron-jobs/createFileLists +++ b/cron-jobs/createFileLists @@ -3,7 +3,6 @@ reposdir=/srv/ftp/ targetdir=$reposdir repos="core extra testing community" -DB_COMPRESSION="gz" . $(dirname $0)/../config @@ -30,16 +29,18 @@ getpkgname() { echo $tmp } -case "${DB_COMPRESSION}" in - gz) TAR_OPT="z" ;; - bz2) TAR_OPT="j" ;; - xz) TAR_OPT="J" ;; - *) echo "${DB_COMPRESSION} is not a valid archive compression" && exit 1 ;; +case "${DBEXT}" in + *.gz) TAR_OPT="z" ;; + *.bz2) TAR_OPT="j" ;; + *.xz) TAR_OPT="J" ;; + *) echo "Unknown compression type for DBEXT=${DBEXT}" && exit 1 ;; esac +FILESEXT="${DBEXT//db/files}" + cd $reposdir for repo in $repos; do - REPO_DB_FILE=${repo}.files.tar.${DB_COMPRESSION} + REPO_DB_FILE=${repo}$FILESEXT for arch in ${ARCHES[@]}; do repodir=${repo}/os/${arch}/ cached="no" @@ -77,7 +78,7 @@ for repo in $repos; do cd ${TMPDIR}/${repodir} [ -f "${pkgdir}${REPO_DB_FILE}.old" ] && rm "${pkgdir}${REPO_DB_FILE}.old" [ -f "${pkgdir}${REPO_DB_FILE}" ] && mv "${pkgdir}${REPO_DB_FILE}" "${pkgdir}${REPO_DB_FILE}.old" - bsdtar --exclude=*.tar.${DB_COMPRESSION} -c${TAR_OPT}f ${pkgdir}${REPO_DB_FILE} * + bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f ${pkgdir}${REPO_DB_FILE} * fi cd $reposdir -- cgit v1.2.3-2-g168b From c7c797f0863c41f1e7ef9e17b408dab84ef493e6 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 18 Aug 2009 16:12:59 -0700 Subject: Use db-functions file everywhere This source the config file and gets us functions such as getpkgname which were duplicated elsewhere Signed-off-by: Aaron Griffin --- cron-jobs/createFileLists | 13 +------------ cron-jobs/ftpdir-cleanup | 2 +- cron-jobs/sourceballs | 14 +------------- 3 files changed, 3 insertions(+), 26 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists index 4ad4c22..345f9aa 100755 --- a/cron-jobs/createFileLists +++ b/cron-jobs/createFileLists @@ -4,7 +4,7 @@ reposdir=/srv/ftp/ targetdir=$reposdir repos="core extra testing community" -. $(dirname $0)/../config +. "$(dirname $0)/../db-functions" if [ -f "/tmp/createFileList.lock" ]; then echo "Error: createFileList allready in progress." @@ -18,17 +18,6 @@ CACHEDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1 #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null -getpkgname() { - local tmp - - tmp=${1##*/} - tmp=${tmp%$PKGEXT} - tmp=${tmp%-i686} - tmp=${tmp%-x86_64} - tmp=${tmp%-any} - echo $tmp -} - case "${DBEXT}" in *.gz) TAR_OPT="z" ;; *.bz2) TAR_OPT="j" ;; diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 1a9fd98..04caf93 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -28,7 +28,7 @@ trap ctrl_c 2 /usr/bin/renice +10 -p $$ > /dev/null #Get our destination dir -. $(dirname $0)/../config +. "$(dirname $0)/../db-functions" for repo in $repos; do $(dirname $0)/../misc-scripts/ftpdir-cleanup $repo $CLEANUP_DESTDIR diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index a34ae2d..02ebc66 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -4,7 +4,7 @@ ftpbase="/srv/ftp" srcbase="/srv/ftp/sources" repos="core extra testing community" -. $(dirname $0)/../config +. "$(dirname $0)/../db-functions" LOCKFILE="/tmp/.sourceball.lock" @@ -30,18 +30,6 @@ trap ctrl_c 2 dirname="$(/bin/readlink -f $(/usr/bin/dirname $0))" -getpkgname() { - local tmp - - tmp=${1##*/} - tmp=${tmp%$PKGEXT} - for a in ${ARCHES[@]}; do - tmp=${tmp%-$a} - done - echo ${tmp%-*-*} -} - - FAILED_PKGS="" [ -e "$srcbase/errors.txt" ] && /bin/mv "$srcbase/errors.txt" "$srcbase/errors.txt.old" -- cgit v1.2.3-2-g168b From b40d217f2baf30f5d7c5979618ff940e43636007 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 18 Aug 2009 16:18:22 -0700 Subject: Replace pkgname with pkgbase, for split packages Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 02ebc66..4353935 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -45,26 +45,26 @@ for repo in $repos; do fi cd $ftppath for pkg in *$PKGEXT; do - pkgname=$(getpkgname $pkg) + pkgbase=$(getpkgbase $pkg) srcpath="$srcbase/" srcpkg="${pkg//$PKGEXT/$SRCEXT}" srcpkg="${srcpkg//-$arch/}" #Don't do anything for package in this 'blacklist' - if grep "^$pkgname\$" "$dirname/sourceballs.skip" >/dev/null 2>&1; then + if grep "^$pkgbase\$" "$dirname/sourceballs.skip" >/dev/null 2>&1; then continue fi #Use this file to 'whitelist' or force building some sourceballs, # skipping the license check force="" - if grep "^$pkgname\$" "$dirname/sourceballs.force" >/dev/null 2>&1; then + if grep "^$pkgbase\$" "$dirname/sourceballs.force" >/dev/null 2>&1; then force="-f" fi if [ ! -f "$srcpath$srcpkg" ]; then if ! $dirname/../misc-scripts/make-sourceball $force \ - $pkgname $repo $arch 2>>"$srcbase/errors.txt"; then + $pkgbase $repo $arch 2>>"$srcbase/errors.txt"; then FAILED_PKGS="$FAILED_PKGS $pkg" fi fi -- cgit v1.2.3-2-g168b From d455a0b2cd72cd08b7b6cde878a29841f77a041e Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Fri, 28 Aug 2009 01:27:11 +0200 Subject: check_packages : add support for split packages. I just found a way to support split packages, by using $(type package_${pkg}), parsing that output and running eval on the relevant lines. This is a bit ugly, and while it works fine on my machine and my current abs tree, I cannot guarantee this code is bug free :) Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/check_packages.py | 5 +- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 82 +++++++++++++++++++++++----- 2 files changed, 73 insertions(+), 14 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index 9bdd9c1..d42feab 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -38,6 +38,7 @@ checked_deps = [] class PacmanPackage: def __init__(self): self.name,self.version = "","" + self.base = "" self.path,self.repo = "","" self.deps,self.makedeps = [],[] self.provides,self.conflicts = [],[] @@ -77,6 +78,8 @@ def parse_data(repo,data): if packages.has_key(pkg.name): dup = packages[pkg.name] packages[pkg.name] = pkg + elif attrname == "base": + pkg.base = line elif attrname == "version": pkg.version = line elif attrname == "path": @@ -356,7 +359,7 @@ for name,pkg in packages.iteritems(): print "==> checking mismatches" for name,pkg in repopkgs.iteritems(): pkgdirname = pkg.path.split("/")[-1] - if name != pkgdirname: + if name != pkgdirname and pkg.base != pkgdirname: mismatches.append(name + " vs. " + pkg.path) print "==> checking archs" diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index 7dcfbbb..0faa29f 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -5,24 +5,36 @@ exit() { return; } -parse() { - unset pkgname pkgver pkgrel - unset depends makedepends conflicts provides - ret=0 - dir=$1 - pkgbuild=$dir/PKGBUILD - source $pkgbuild &>/dev/null || ret=$? +variables=('pkgname' 'pkgver' 'pkgrel' 'depends' 'makedepends' 'provides' 'conflicts' ) +readonly -a variables - # ensure $pkgname and $pkgver variables were found - if [ $ret -ne 0 -o -z "$pkgname" -o -z "$pkgver" ]; then - echo -e "%INVALID%\n$pkgbuild\n" - return 1 - fi +backup_package_variables() { + for var in ${variables[@]}; do + indirect="${var}_backup" + eval "${indirect}=(\${$var[@]})" + done +} + +restore_package_variables() { + for var in ${variables[@]}; do + indirect="${var}_backup" + if [ -n "${!indirect}" ]; then + eval "${var}=(\${$indirect[@]})" + else + unset ${var} + fi + done +} +print_info() { echo -e "%NAME%\n$pkgname\n" echo -e "%VERSION%\n$pkgver-$pkgrel\n" echo -e "%PATH%\n$dir\n" + if [ -n "$pkgbase" ]; then + echo -e "%BASE%\n$pkgbase\n" + fi + if [ -n "$arch" ]; then echo "%ARCH%" for i in ${arch[@]}; do echo $i; done @@ -52,6 +64,50 @@ parse() { for i in ${provides[@]}; do echo $i; done echo "" fi +} + +source_pkgbuild() { + ret=0 + dir=$1 + pkgbuild=$dir/PKGBUILD + for var in ${variables[@]}; do + unset ${var} + done + source $pkgbuild &>/dev/null || ret=$? + + # ensure $pkgname and $pkgver variables were found + if [ $ret -ne 0 -o -z "$pkgname" -o -z "$pkgver" ]; then + echo -e "%INVALID%\n$pkgbuild\n" + return 1 + fi + + if [ "${#pkgname[@]}" -gt "1" ]; then + for pkg in ${pkgname[@]}; do + if [ "$(type -t package_${pkg})" != "function" ]; then + echo -e "%INVALID%\n$pkgbuild\n" + return 1 + else + backup_package_variables + pkgname=$pkg + while IFS= read -r line; do + var=${line%%=*} + var="${var#"${var%%[![:space:]]*}"}" # remove leading whitespace characters + for realvar in ${variables[@]}; do + if [ "$var" == "$realvar" ]; then + eval $line + break + fi + done + done < <(type package_${pkg}) + print_info + restore_package_variables + fi + done + else + echo + print_info + fi + return 0 } @@ -63,7 +119,7 @@ find_pkgbuilds() { fi if [ -f $1/PKGBUILD ]; then - parse $1 + source_pkgbuild $1 return fi empty=1 -- cgit v1.2.3-2-g168b From 620b631fedbbd3596baa6752b9234249a2239a2a Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Fri, 28 Aug 2009 01:27:12 +0200 Subject: add deplist to hierarchy problems This should help a lot figuring out whether a given hierarchy problem is fixable (if it doesn't require moving too many deps). example output core/crda depends on extra/python-m2crypto (27 extra (make)deps to pull) core/iputils depends on extra/opensp (29 extra (make)deps to pull) core/iputils depends on extra/libxslt (25 extra (make)deps to pull) core/iputils depends on extra/docbook-xsl (27 extra (make)deps to pull) core/udev depends on extra/gperf (0 extra (make)deps to pull) core/udev depends on extra/libxslt (25 extra (make)deps to pull) core/e2fsprogs depends on extra/bc (0 extra (make)deps to pull) core/sqlite3 depends on extra/tcl (0 extra (make)deps to pull) core/ca-certificates depends on extra/ruby (25 extra (make)deps to pull) the actual deps are only displayed when there are less than 10. Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/check_packages.py | 50 +++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index d42feab..f2a9601 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -182,11 +182,50 @@ def verify_deps(name,repo,deps): break if not pkgdep: pkgdep = pkglist[0] - hierarchy.append(repo + "/" + name + " depends on " + pkgdep.repo + "/" + pkgdep.name) + hierarchy.append((repo,name,pkgdep)) + pkg_deps.append(pkgdep) return (pkg_deps,missdeps,hierarchy) +def compute_deplist_aux(pkg,deplist): + newdeplist = [] + list = [] + if pkgdeps.has_key(pkg): + list.extend(pkgdeps[pkg]) + if makepkgdeps.has_key(pkg): + list.extend(makepkgdeps[pkg]) + for dep in list: + if dep not in deplist: + newdeplist.append(dep) + deplist.append(dep) + for dep in newdeplist: + deplist2 = compute_deplist_aux(dep,deplist) + for dep2 in deplist2: + if dep2 not in deplist: + deplist.append(dep2) + return deplist + +def compute_deplist(pkg): + return compute_deplist_aux(pkg,[]) + +def check_hierarchy(deph): + hierarchy = [] + for (repo,name,pkgdep) in deph: + deplist = compute_deplist(pkgdep) + valid_repos = get_repo_hierarchy(repo) + extdeps = [] + for dep in deplist: + if dep.repo not in valid_repos: + extdeps.append(dep.name) + string = repo + "/" + name + " depends on " + pkgdep.repo + "/" + pkgdep.name + " (" + string += "%s extra (make)deps to pull" % len(extdeps) + if 0 < len(extdeps) < 10: + string += " : " + ' '.join(extdeps) + string += ")" + hierarchy.append(string) + return hierarchy + def get_repo_hierarchy(repo): repo_hierarchy = {'core': ['core'], \ 'extra': ['core', 'extra'], \ @@ -373,19 +412,24 @@ for name,pkg in packages.iteritems(): pkg.deps = [dep for dep in pkg.deps if not p.match(dep)] pkg.makedeps = [dep for dep in pkg.makedeps if not p.match(dep)] +deph,makedeph = [],[] + print "==> checking dependencies" for name,pkg in repopkgs.iteritems(): (deps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.deps) pkgdeps[pkg] = deps missing_deps.extend(missdeps) - dep_hierarchy.extend(hierarchy) + deph.extend(hierarchy) print "==> checking makedepends" for name,pkg in repopkgs.iteritems(): (makedeps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.makedeps) makepkgdeps[pkg] = makedeps missing_makedeps.extend(missdeps) - makedep_hierarchy.extend(hierarchy) + makedeph.extend(hierarchy) + +dep_hierarchy = check_hierarchy(deph) +makedep_hierarchy = check_hierarchy(makedeph) print "==> checking for circular dependencies" # make sure pkgdeps is filled for every package -- cgit v1.2.3-2-g168b From 1e84b6715a688e1c2f1e50e3834511ed21d5c291 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 1 Sep 2009 09:34:21 -0700 Subject: sourceballs: explicitly add the 'any' arch Loop over all arches AND the 'any' arch. Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 4353935..aa06aff 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -36,7 +36,7 @@ FAILED_PKGS="" echo "Errors occured during run:" > "$srcbase/errors.txt" for repo in $repos; do - for arch in ${ARCHES[@]}; do + for arch in ${ARCHES[@]} any; do export CARCH="$arch" ftppath="$ftpbase/$repo/os/$arch" if [ ! -d "$ftppath" ]; then -- cgit v1.2.3-2-g168b From 5ebf55bedfa891a80170facc79f2fbfca11983c4 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 8 Sep 2009 16:39:58 -0400 Subject: Added sourceballs.force entries Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs.force | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs.force b/cron-jobs/sourceballs.force index e69de29..badf15d 100644 --- a/cron-jobs/sourceballs.force +++ b/cron-jobs/sourceballs.force @@ -0,0 +1,4 @@ +faad2 +wxgtk +wxpython +glhack -- cgit v1.2.3-2-g168b From 2afa1063284a25ae2691b2727c45238d15025a9a Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 13 Sep 2009 18:16:55 -0500 Subject: Be more careful with config file inclusion We can't really do the basedir magic from db-functions as it is just being sourced and is not the currently executing script. Although a bit repetitive, it is a lot safer to just include the config file everywhere. Noticed this when trying to run the latest available scripts on sigurd. Signed-off-by: Dan McGee --- cron-jobs/createFileLists | 1 + cron-jobs/sourceballs | 1 + 2 files changed, 2 insertions(+) (limited to 'cron-jobs') diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists index 345f9aa..d936853 100755 --- a/cron-jobs/createFileLists +++ b/cron-jobs/createFileLists @@ -5,6 +5,7 @@ targetdir=$reposdir repos="core extra testing community" . "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" if [ -f "/tmp/createFileList.lock" ]; then echo "Error: createFileList allready in progress." diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index aa06aff..46c2fa3 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -5,6 +5,7 @@ srcbase="/srv/ftp/sources" repos="core extra testing community" . "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" LOCKFILE="/tmp/.sourceball.lock" -- cgit v1.2.3-2-g168b From 0897913c0adf10a0a9ff4d8790842715e0e3b1b4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 13 Sep 2009 18:19:59 -0500 Subject: Add new get_repos_for_host() function And use it. This allows us to have server-specific behavior in our scripts without further patching, and it also allows us to simplify some of our scripts a fair amount. Signed-off-by: Dan McGee --- cron-jobs/adjust-permissions | 50 ++++++++++++++++++++++++-------------------- cron-jobs/ftpdir-cleanup | 8 +++---- 2 files changed, 31 insertions(+), 27 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index 6670a6f..91f2de7 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -1,36 +1,40 @@ -#!/bin/sh +#!/bin/bash if [ -f /tmp/.ftpmaint.lck ]; then - exit 0 + exit 0 fi /bin/touch /tmp/.ftpmaint.lck +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" + +get_dir_owner() { + case $1 in + core) + echo "ftp:ftp-arch" ;; + extra) + echo "ftp:ftp-extra" ;; + testing) + echo "ftp:ftp-extra" ;; + community) + echo "root:tusers" ;; + community-testing) + echo "root:tusers" ;; + esac +} + #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null cd /srv/ftp -if [ -d "core" ]; then - #This is unique to gerolde (main arch server) - /bin/chown -R ftp:ftp-arch core/os/any - /bin/chown -R ftp:ftp-arch core/os/i686 - /bin/chown -R ftp:ftp-arch core/os/x86_64 - /bin/chown -R ftp:ftp-extra {extra,testing}/os/any - /bin/chown -R ftp:ftp-extra {extra,testing}/os/i686 - /bin/chown -R ftp:ftp-extra {extra,testing}/os/x86_64 - for d in core extra testing; do - /bin/chmod -R g+w $d/os/any - /bin/chmod -R g+w $d/os/i686 - /bin/chmod -R g+w $d/os/x86_64 - done -else - /bin/chown -R root:tusers {community,community-testing}/os/any - /bin/chown -R root:tusers {community,community-testing}/os/i686 - /bin/chown -R root:tusers {community,community-testing}/os/x86_64 - /bin/chmod -R g+w {community,community-testing}/os/any - /bin/chmod -R g+w {community,community-testing}/os/i686 - /bin/chmod -R g+w {community,community-testing}/os/x86_64 -fi +for d in $(get_repos_for_host); do + owner="$(get_dir_owner $d)" + echo /bin/chown -R $owner $d/os/{any,i686,x86_64} + #/bin/chown -R $owner $d/os/{any,i686,x86_64} + echo /bin/chmod -R g+w $d/os/{any,i686,x86_64} + #/bin/chmod -R g+w $d/os/{any,i686,x86_64} +done /bin/chmod 555 /srv/ftp diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 04caf93..d66138c 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -1,6 +1,9 @@ #!/bin/bash -repos="core extra testing" +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" + +repos="$(get_repos_for_host)" LOCKFILE="/tmp/.ftpdircleanup.lock" @@ -27,9 +30,6 @@ trap ctrl_c 2 #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null -#Get our destination dir -. "$(dirname $0)/../db-functions" - for repo in $repos; do $(dirname $0)/../misc-scripts/ftpdir-cleanup $repo $CLEANUP_DESTDIR done -- cgit v1.2.3-2-g168b From 326d22b63d14012da72e2c9b56109c490e98cf63 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 13 Sep 2009 18:27:28 -0500 Subject: Maybe I should make the script really adjust permissions Signed-off-by: Dan McGee --- cron-jobs/adjust-permissions | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index 91f2de7..0f87075 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -30,10 +30,8 @@ get_dir_owner() { cd /srv/ftp for d in $(get_repos_for_host); do owner="$(get_dir_owner $d)" - echo /bin/chown -R $owner $d/os/{any,i686,x86_64} - #/bin/chown -R $owner $d/os/{any,i686,x86_64} - echo /bin/chmod -R g+w $d/os/{any,i686,x86_64} - #/bin/chmod -R g+w $d/os/{any,i686,x86_64} + /bin/chown -R $owner $d/os/{any,i686,x86_64} + /bin/chmod -R g+w $d/os/{any,i686,x86_64} done /bin/chmod 555 /srv/ftp -- cgit v1.2.3-2-g168b From a46bca9a39b37e7574f87468ddc5fb9b9577786e Mon Sep 17 00:00:00 2001 From: Henning Garus Date: Tue, 1 Sep 2009 23:54:47 +0200 Subject: check_packages.py: Allow multiple abs-trees By parsing multiple abs trees we can add any when parsing the other trees, checking any standalone doesn't make much sense. Signed-off-by: Henning Garus Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/check_packages.py | 32 +++++++++++++++-------------- cron-jobs/integrity-check | 2 +- 2 files changed, 18 insertions(+), 16 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index f2a9601..c12858d 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -51,10 +51,11 @@ class Depend: self.mod = mod def parse_pkgbuilds(repos,arch): - for repo in repos: - data = commands.getoutput(os.path.dirname(sys.argv[0]) + '/parse_pkgbuilds.sh ' - + arch + ' ' + absroot + '/' + repo) - parse_data(repo,data) + for absroot in absroots: + for repo in repos: + data = commands.getoutput(os.path.dirname(sys.argv[0]) + '/parse_pkgbuilds.sh ' + + arch + ' ' + absroot + '/' + repo) + parse_data(repo,data) def parse_data(repo,data): attrname = None @@ -332,9 +333,9 @@ def print_usage(): print "Usage: ./check_packages.py [OPTION]" print "" print "Options:" - print " --abs-tree= Check the specified tree (default : /var/abs)" + print " --abs-tree= Check the specified tree(s) (default : /var/abs)" print " --repos= Check the specified repos (default : core,extra)" - print " --arch= Check the specified arch (default : i686)" + print " --arch= Check the specified arch (default : i686)" print " -h, --help Show this help and exit" print "" print "Examples:" @@ -345,7 +346,7 @@ def print_usage(): print "" ## Default path to the abs root directory -absroot = "/var/abs" +absroots = ["/var/abs"] ## Default list of repos to check repos = ['core', 'extra'] ## Default arch @@ -359,7 +360,7 @@ except getopt.GetoptError: if opts != []: for o, a in opts: if o in ("--abs-tree"): - absroot = a + absroot = a.split(',') elif o in ("--repos"): repos = a.split(",") elif o in ("--arch"): @@ -371,14 +372,15 @@ if opts != []: print_usage() sys.exit() -if not os.path.isdir(absroot): - print "Error : the abs tree " + absroot + " does not exist" - sys.exit() -for repo in repos: - repopath = absroot + "/" + repo - if not os.path.isdir(repopath): - print "Error : the repository " + repo + " does not exist in " + absroot +for absroot in absroots: + if not os.path.isdir(absroot): + print "Error : the abs tree " + absroot + " does not exist" sys.exit() + for repo in repos: + repopath = absroot + "/" + repo + if not os.path.isdir(repopath): + print "Error : the repository " + repo + " does not exist in " + absroot + sys.exit() # repos which need to be loaded loadrepos = set([]) for repo in repos: diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index 0b59064..b3185ec 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -9,5 +9,5 @@ fi $basedir/check_archlinux/check_packages.py \ --repos="$1" \ - --abs-tree="/srv/abs/rsync/$2" --arch="$2" |\ + --abs-tree="/srv/abs/rsync/$2,/srv/abs/rsync/any" --arch="$2" |\ $basedir/devlist-mailer "Integrity Check $2: $1" "$3" -- cgit v1.2.3-2-g168b From ee0dd1ef6e9a0169ad54947758f2d7858a60d960 Mon Sep 17 00:00:00 2001 From: Henning Garus Date: Tue, 1 Sep 2009 23:57:39 +0200 Subject: Add db comparison to integrity check Compare the abs tree with the repo dbs to check if we have a PKGBUILD for each package in the dbs and vice versa. Signed-off-by: Henning Garus Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/check_packages.py | 55 +++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index c12858d..feab987 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -16,9 +16,11 @@ # a non-core package) # 8. Circular dependencies -import os,re,commands,getopt,sys,alpm +import os,re,commands,getopt,sys,tarfile,alpm import pdb +DBEXT='.db.tar.gz' + packages = {} # pkgname : PacmanPackage provisions = {} # provision : PacmanPackage pkgdeps,makepkgdeps = {},{} # pkgname : list of the PacmanPackage dependencies @@ -26,6 +28,9 @@ invalid_pkgbuilds = [] missing_pkgbuilds = [] dups = [] +dbonly = [] +absonly = [] + mismatches = [] missing_deps = [] missing_makedeps = [] @@ -102,6 +107,17 @@ def parse_data(repo,data): provisions[provname] = [] provisions[provname].append(pkg) +def parse_dbs(repos,arch): + dbpkgs = {} + for repo in repos: + pkgs = set([]) + db = tarfile.open(os.path.join(repodir,repo,'os',arch,repo + DBEXT)) + for line in db.getnames(): + if not '/' in line: + pkgs.add(line.rsplit('-',2)[0]) + dbpkgs[repo] = pkgs + return(dbpkgs) + def splitdep(dep): name = dep version = "" @@ -317,6 +333,8 @@ def print_results(): print_result(dep_hierarchy, "Repo Hierarchy for Dependencies") print_result(makedep_hierarchy, "Repo Hierarchy for Makedepends") print_result(circular_deps, "Circular Dependencies") + print_result(dbonly, "Packages found in db, but not in tree") + print_result(absonly,"Packages found in tree, but not in db") print_subheading("Summary") print "Missing PKGBUILDs: ", len(missing_pkgbuilds) print "Invalid PKGBUILDs: ", len(invalid_pkgbuilds) @@ -326,6 +344,8 @@ def print_results(): print "Missing (make)dependencies: ", len(missing_deps)+len(missing_makedeps) print "Repo hierarchy problems: ", len(dep_hierarchy)+len(makedep_hierarchy) print "Circular dependencies: ", len(circular_deps) + print "In db, but not in tree: ", len(dbonly) + print "In tree, but not in db: ", len(absonly) print "" def print_usage(): @@ -336,6 +356,7 @@ def print_usage(): print " --abs-tree= Check the specified tree(s) (default : /var/abs)" print " --repos= Check the specified repos (default : core,extra)" print " --arch= Check the specified arch (default : i686)" + print " --repo-dir= Check the dbs at the specified path (default : /srv/ftp)" print " -h, --help Show this help and exit" print "" print "Examples:" @@ -351,9 +372,12 @@ absroots = ["/var/abs"] repos = ['core', 'extra'] ## Default arch arch = "i686" +## Default repodir +repodir = "/srv/ftp" try: - opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos=", "arch="]) + opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos=", + "arch=", "repo-dir="]) except getopt.GetoptError: print_usage() sys.exit() @@ -365,6 +389,8 @@ if opts != []: repos = a.split(",") elif o in ("--arch"): arch = a + elif o in ("--repo-dir"): + repodir = a else: print_usage() sys.exit() @@ -381,6 +407,17 @@ for absroot in absroots: if not os.path.isdir(repopath): print "Error : the repository " + repo + " does not exist in " + absroot sys.exit() +if not os.path.isdir(repodir): + print "Error: the repository directory %s does not exist" % repodir + sys.exit() +for repo in repos: + path = os.path.join(repodir,repo,'os',arch,repo + DBEXT) + if not os.path.isfile(path): + print "Error : repo DB %s : File not found" % path + sys.exit() + if not tarfile.is_tarfile(path): + print "Error : Cant open repo DB %s, not a valid tar file" % path + sys.exit() # repos which need to be loaded loadrepos = set([]) for repo in repos: @@ -397,6 +434,9 @@ for name,pkg in packages.iteritems(): if pkg.repo in repos: repopkgs[name] = pkg +print "==> parsing db files" +dbpkgs = parse_dbs(repos,arch) + print "==> checking mismatches" for name,pkg in repopkgs.iteritems(): pkgdirname = pkg.path.split("/")[-1] @@ -441,4 +481,15 @@ for name,pkg in packages.iteritems(): pkgdeps[pkg] = deps find_scc(repopkgs.values()) +print "==> checking for differences between db files and pkgbuilds" +for repo in repos: + for pkg in dbpkgs[repo]: + if not (pkg in repopkgs.keys() and repopkgs[pkg].repo == repo): + dbonly.append("%s/%s" % (repo,pkg)) +dbonly.sort() +for name,pkg in repopkgs.iteritems(): + if not name in dbpkgs[pkg.repo]: + absonly.append("%s/%s" % (pkg.repo,name)) +absonly.sort() + print_results() -- cgit v1.2.3-2-g168b From ea827b50d0c65b4ba9f2e2d3c0761572915e6b03 Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Wed, 2 Sep 2009 18:29:19 +0200 Subject: add the repo/ info to most results, and sort them Add repo information to most results by using the repo/pkgname syntax (instead of only pkgname) Sort all results list (with the point above, this sorts the results by repo :)) Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/check_packages.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index feab987..96f3181 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -189,7 +189,7 @@ def verify_deps(name,repo,deps): for dep in deps: pkglist = verify_dep(dep) if pkglist == []: - missdeps.append(name + " --> '" + dep + "'") + missdeps.append(repo + "/" + name + " --> '" + dep + "'") else: valid_repos = get_repo_hierarchy(repo) pkgdep = None @@ -252,12 +252,12 @@ def get_repo_hierarchy(repo): else: return ['core','extra','community'] -def verify_archs(name,archs): +def verify_archs(name,repo,archs): valid_archs = ['any', 'i686', 'x86_64'] invalid_archs = [] for arch in archs: if arch not in valid_archs: - invalid_archs.append(name + " --> " + arch) + invalid_archs.append(repo + "/" + name + " --> " + arch) return invalid_archs def find_scc(packages): @@ -295,7 +295,7 @@ def tarjan(pkg): return path = pkg.name while pkg != dep: - path = dep.name + ">" + path + path = dep.repo + "/" + dep.name + ">" + path dep = S.pop() path = dep.name + ">" + path if pkg.repo in repos: @@ -318,6 +318,7 @@ def print_missdeps(pkgname,missdeps) : def print_result(list, subheading): if len(list) > 0: + list.sort() print_subheading(subheading) for item in list: print item @@ -445,7 +446,7 @@ for name,pkg in repopkgs.iteritems(): print "==> checking archs" for name,pkg in repopkgs.iteritems(): - archs = verify_archs(name,pkg.archs) + archs = verify_archs(name,pkg.repo,pkg.archs) invalid_archs.extend(archs) # ugly hack to strip the weird kblic- deps @@ -486,10 +487,8 @@ for repo in repos: for pkg in dbpkgs[repo]: if not (pkg in repopkgs.keys() and repopkgs[pkg].repo == repo): dbonly.append("%s/%s" % (repo,pkg)) -dbonly.sort() for name,pkg in repopkgs.iteritems(): if not name in dbpkgs[pkg.repo]: absonly.append("%s/%s" % (pkg.repo,name)) -absonly.sort() print_results() -- cgit v1.2.3-2-g168b From fe4152a6949a9ec3132d1a1c08335f370989198f Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Thu, 3 Sep 2009 22:34:39 +0200 Subject: improve compute_dep algorithm The compute_dep function I wrote for the more informative hierarchy output was very inefficient. It's much better now (10s -> 0.5s) on my box, and I get exactly the same results :) Now the big majority of the time is again spent on parsing pkgbuilds. Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/check_packages.py | 35 +++++++++++++---------------- 1 file changed, 16 insertions(+), 19 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index 96f3181..9ffdac5 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -205,26 +205,22 @@ def verify_deps(name,repo,deps): return (pkg_deps,missdeps,hierarchy) -def compute_deplist_aux(pkg,deplist): - newdeplist = [] - list = [] - if pkgdeps.has_key(pkg): - list.extend(pkgdeps[pkg]) - if makepkgdeps.has_key(pkg): - list.extend(makepkgdeps[pkg]) - for dep in list: - if dep not in deplist: - newdeplist.append(dep) - deplist.append(dep) - for dep in newdeplist: - deplist2 = compute_deplist_aux(dep,deplist) - for dep2 in deplist2: - if dep2 not in deplist: - deplist.append(dep2) - return deplist - def compute_deplist(pkg): - return compute_deplist_aux(pkg,[]) + list = [] + stack = [pkg] + while stack != []: + dep = stack.pop() + if pkgdeps.has_key(dep): + for dep2 in pkgdeps[dep]: + if dep2 not in list: + list.append(dep2) + stack.append(dep2) + if makepkgdeps.has_key(dep): + for dep2 in makepkgdeps[dep]: + if dep2 not in list: + list.append(dep2) + stack.append(dep2) + return list def check_hierarchy(deph): hierarchy = [] @@ -471,6 +467,7 @@ for name,pkg in repopkgs.iteritems(): missing_makedeps.extend(missdeps) makedeph.extend(hierarchy) +print "==> checking hierarchy" dep_hierarchy = check_hierarchy(deph) makedep_hierarchy = check_hierarchy(makedeph) -- cgit v1.2.3-2-g168b From b36b754497df69778273f6a6e7b0d91679646565 Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Thu, 3 Sep 2009 23:34:24 +0200 Subject: parse_pkgbuilds : simple performance tweak this simple tweak gives a nice perf boost : from 10s to 7s to parse extra repo. indeed calling basename caused a fork to happen for the thousand of files being considered Now the major bottleneck is parsing split pkgbuilds, but I am afraid there is no magical solution for that :( Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index 0faa29f..c2df7d1 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -113,7 +113,7 @@ source_pkgbuild() { find_pkgbuilds() { #Skip over some dirs - local d="$(basename $1)" + local d="${1##*/}" if [ "$d" = "CVS" -o "$d" = ".svn" ]; then return fi -- cgit v1.2.3-2-g168b From 7eccf95bb5d53cb1b4f8c1d363cd5645ca4e567e Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Sun, 6 Sep 2009 21:36:42 +0200 Subject: improve the way parse_pkgbuilds.sh is called 1) check the return value. before a failed call to parse_pkgbuilds.sh was completely silent.. not good :) 2) call realpath on check_packages.py before determining the directory name, to figure out where parse_pkgbuilds.sh is. This allows to symlink check_packages.py and use the symlink without moving parse_pkgbuilds.sh . for example : /foo/bar/ contains check_packages.py and parse_pkgbuilds.sh ln -s /foo/bar/check_packages.py /bin/check_package dirname(/bin/check_package) returns /bin/ so not good dirname(realpath(/bin/check_package)) returns /foo/bar/ so good Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/check_packages.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index 9ffdac5..cb7eae4 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -58,9 +58,13 @@ class Depend: def parse_pkgbuilds(repos,arch): for absroot in absroots: for repo in repos: - data = commands.getoutput(os.path.dirname(sys.argv[0]) + '/parse_pkgbuilds.sh ' - + arch + ' ' + absroot + '/' + repo) - parse_data(repo,data) + cmd = os.path.dirname(os.path.realpath(sys.argv[0])) + '/parse_pkgbuilds.sh ' + cmd += arch + ' ' + absroot + '/' + repo + (status,output) = commands.getstatusoutput(cmd) + if status != 0: + print "Error : failed to run '%s'" % cmd + sys.exit() + parse_data(repo,output) def parse_data(repo,data): attrname = None -- cgit v1.2.3-2-g168b From 12aba89ed2100552ddfe7d69bbd9e35a2019d098 Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Tue, 15 Sep 2009 09:19:02 +0200 Subject: parse_pkgbuilds.sh : fix pkgbase handling add the default value for pkgbase also rework slightly how variables are handled (better distinction between splitpkg variables and the rest) Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index c2df7d1..5cd17e4 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -5,18 +5,19 @@ exit() { return; } -variables=('pkgname' 'pkgver' 'pkgrel' 'depends' 'makedepends' 'provides' 'conflicts' ) -readonly -a variables +splitpkg_overrides=('depends' 'optdepends' 'provides' 'conflicts') +variables=('pkgname' 'pkgbase' 'pkgver' 'pkgrel' 'makedepends' 'arch' ${splitpkg_overrides[@]}) +readonly -a variables splitpkg_overrides backup_package_variables() { - for var in ${variables[@]}; do + for var in ${splitpkg_overrides[@]}; do indirect="${var}_backup" eval "${indirect}=(\${$var[@]})" done } restore_package_variables() { - for var in ${variables[@]}; do + for var in ${splitpkg_overrides[@]}; do indirect="${var}_backup" if [ -n "${!indirect}" ]; then eval "${var}=(\${$indirect[@]})" @@ -82,6 +83,7 @@ source_pkgbuild() { fi if [ "${#pkgname[@]}" -gt "1" ]; then + pkgbase=${pkgbase:-${pkgname[0]}} for pkg in ${pkgname[@]}; do if [ "$(type -t package_${pkg})" != "function" ]; then echo -e "%INVALID%\n$pkgbuild\n" -- cgit v1.2.3-2-g168b From 43f4ada2f207ebf6d3cdd95d511e3ecdc01e6d7a Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 18 Sep 2009 17:22:55 -0400 Subject: Bump check_archlinux module for libalpm.so.4 Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/alpm.so | Bin 0 -> 7238 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100755 cron-jobs/check_archlinux/alpm.so (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux/alpm.so b/cron-jobs/check_archlinux/alpm.so new file mode 100755 index 0000000..b87b894 Binary files /dev/null and b/cron-jobs/check_archlinux/alpm.so differ -- cgit v1.2.3-2-g168b From 85eee9a95d2fcbdce483b94202b489d07d258690 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 18 Sep 2009 17:28:46 -0400 Subject: check_archlinux: absroot -> absroots Allow us to use the --abs-root flag. He he Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/check_packages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index cb7eae4..25362c0 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -385,7 +385,7 @@ except getopt.GetoptError: if opts != []: for o, a in opts: if o in ("--abs-tree"): - absroot = a.split(',') + absroots = a.split(',') elif o in ("--repos"): repos = a.split(",") elif o in ("--arch"): -- cgit v1.2.3-2-g168b From 496fc1c7f54132410af5fd84bfe10b8e68d02a1e Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 18 Sep 2009 17:33:26 -0400 Subject: integrity-check: Loop over arches THis simplifies the command line and we do both anyway Signed-off-by: Aaron Griffin --- cron-jobs/integrity-check | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index b3185ec..8f23129 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -3,11 +3,13 @@ basedir="$(dirname $0)" if [ $# -ne 3 ]; then - echo "usage: $(basename $0) [,,...] " >&2 + echo "usage: $(basename $0) [,,...] " >&2 exit 1 fi -$basedir/check_archlinux/check_packages.py \ - --repos="$1" \ - --abs-tree="/srv/abs/rsync/$2,/srv/abs/rsync/any" --arch="$2" |\ - $basedir/devlist-mailer "Integrity Check $2: $1" "$3" +for arch in i686 x86_64; do + $basedir/check_archlinux/check_packages.py \ + --repos="$1" \ + --abs-tree="/srv/abs/rsync/$arch,/srv/abs/rsync/any" --arch="$arch" |\ + $basedir/devlist-mailer "Integrity Check $arch: $1" "$3" +done -- cgit v1.2.3-2-g168b From 17e96ac3d2505ed8841ff9240492e7378472d364 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 18 Sep 2009 18:18:44 -0400 Subject: integrity-check: Forgot to decrement the arg count Signed-off-by: Aaron Griffin --- cron-jobs/integrity-check | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index 8f23129..c2546de 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -2,7 +2,7 @@ basedir="$(dirname $0)" -if [ $# -ne 3 ]; then +if [ $# -ne 2 ]; then echo "usage: $(basename $0) [,,...] " >&2 exit 1 fi -- cgit v1.2.3-2-g168b From d7f1242883aea50023931447c3c2f527528084b3 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 18 Sep 2009 18:26:02 -0400 Subject: integrity-check: Rename $3 -> $2 OMFG such a simple script and I keep missing crap Signed-off-by: Aaron Griffin --- cron-jobs/integrity-check | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index c2546de..0b2b9c7 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -11,5 +11,5 @@ for arch in i686 x86_64; do $basedir/check_archlinux/check_packages.py \ --repos="$1" \ --abs-tree="/srv/abs/rsync/$arch,/srv/abs/rsync/any" --arch="$arch" |\ - $basedir/devlist-mailer "Integrity Check $arch: $1" "$3" + $basedir/devlist-mailer "Integrity Check $arch: $1" "$2" done -- cgit v1.2.3-2-g168b From 4643d931f6adc4f7f55c7c2fbb37532614742c74 Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Sun, 20 Sep 2009 14:50:34 +0200 Subject: check_packages : fix a bug when filling provisions When we had a duplicate package, we would replace the old ref by the new ref. But the provisions dict kept both the old and new ref. This was not good at all. Now, we just keep the old ref, and we only fill provisions after all packages have been parsed. This should be much more sane. Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/check_packages.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index 25362c0..e8830e0 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -22,8 +22,9 @@ import pdb DBEXT='.db.tar.gz' packages = {} # pkgname : PacmanPackage +repopkgs = {} # pkgname : PacmanPackage provisions = {} # provision : PacmanPackage -pkgdeps,makepkgdeps = {},{} # pkgname : list of the PacmanPackage dependencies +pkgdeps,makepkgdeps = {},{} # PacmanPackage : list of the PacmanPackage dependencies invalid_pkgbuilds = [] missing_pkgbuilds = [] dups = [] @@ -87,7 +88,8 @@ def parse_data(repo,data): dup = None if packages.has_key(pkg.name): dup = packages[pkg.name] - packages[pkg.name] = pkg + else: + packages[pkg.name] = pkg elif attrname == "base": pkg.base = line elif attrname == "version": @@ -106,10 +108,6 @@ def parse_data(repo,data): pkg.conflicts.append(line) elif attrname == "provides": pkg.provides.append(line) - provname=line.split("=")[0] - if not provisions.has_key(provname): - provisions[provname] = [] - provisions[provname].append(pkg) def parse_dbs(repos,arch): dbpkgs = {} @@ -278,11 +276,9 @@ def tarjan(pkg): index += 1 checked_deps.append(pkg) S.append(pkg) + deps = [] if pkgdeps.has_key(pkg): deps = pkgdeps[pkg] - else: - print pkg.name - deps = [] for dep in deps: if not pkgindex.has_key(dep): tarjan(dep) @@ -430,7 +426,15 @@ print "\nPerforming integrity checks..." print "==> parsing pkgbuilds" parse_pkgbuilds(loadrepos,arch) -repopkgs = {} +# fill provisions +for name,pkg in packages.iteritems(): + for prov in pkg.provides: + provname=prov.split("=")[0] + if not provisions.has_key(provname): + provisions[provname] = [] + provisions[provname].append(pkg) + +# fill repopkgs for name,pkg in packages.iteritems(): if pkg.repo in repos: repopkgs[name] = pkg -- cgit v1.2.3-2-g168b From c573cb4a00969d8d81d0fdf9789e255f4d296d39 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 21 Sep 2009 16:20:46 -0400 Subject: Temporarily disable community sourceball generation Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 46c2fa3..72baa32 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -2,7 +2,7 @@ ftpbase="/srv/ftp" srcbase="/srv/ftp/sources" -repos="core extra testing community" +repos="core extra testing" #community" . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" -- cgit v1.2.3-2-g168b From 29b56cd5e289a35a24459af70dd6678b0c55cb9b Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 21 Sep 2009 18:21:33 -0400 Subject: Set/restore umask in the sourceballs cron job Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 3 +++ 1 file changed, 3 insertions(+) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 72baa32..0d8be62 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -11,6 +11,7 @@ LOCKFILE="/tmp/.sourceball.lock" cleanup () { rm -f "$LOCKFILE" + restore_umask exit 0 } @@ -27,6 +28,8 @@ fi trap cleanup 0 trap ctrl_c 2 +set_umask + /bin/touch "$LOCKFILE" dirname="$(/bin/readlink -f $(/usr/bin/dirname $0))" -- cgit v1.2.3-2-g168b From 7af26d483b6987e0f3d42b25d8cb4e39f691a0a3 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 21 Sep 2009 19:11:42 -0400 Subject: Add a safety check for existence in pkg loop Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 1 + 1 file changed, 1 insertion(+) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 0d8be62..f285d7f 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -49,6 +49,7 @@ for repo in $repos; do fi cd $ftppath for pkg in *$PKGEXT; do + [ -f "$pkg" ] || continue pkgbase=$(getpkgbase $pkg) srcpath="$srcbase/" srcpkg="${pkg//$PKGEXT/$SRCEXT}" -- cgit v1.2.3-2-g168b From de547b70c1b3f6dfc163d96318d6f915446bbc4c Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 24 Sep 2009 15:01:08 -0700 Subject: Fixed test for existence of split package sources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch also removes the unused CARCH variable. Signed-off-by: Eric Bélanger [Aaron: Broke srcpkgbase logic into srcpkgname/srcpkgbase to make it more readable] Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index f285d7f..09b1df7 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -41,7 +41,6 @@ echo "Errors occured during run:" > "$srcbase/errors.txt" for repo in $repos; do for arch in ${ARCHES[@]} any; do - export CARCH="$arch" ftppath="$ftpbase/$repo/os/$arch" if [ ! -d "$ftppath" ]; then echo "FTP path does not exist: $ftppath" >2 @@ -54,6 +53,8 @@ for repo in $repos; do srcpath="$srcbase/" srcpkg="${pkg//$PKGEXT/$SRCEXT}" srcpkg="${srcpkg//-$arch/}" + srcpkgname="${srcpkg%-*-*$SRCEXT}" + srcpkgbase="${srcpkg/$srcpkgname/$pkgbase}" #Don't do anything for package in this 'blacklist' if grep "^$pkgbase\$" "$dirname/sourceballs.skip" >/dev/null 2>&1; then @@ -67,7 +68,7 @@ for repo in $repos; do force="-f" fi - if [ ! -f "$srcpath$srcpkg" ]; then + if [ ! \( -f "$srcpath$srcpkg" -o -f "$srcpath$srcpkgbase" \) ]; then if ! $dirname/../misc-scripts/make-sourceball $force \ $pkgbase $repo $arch 2>>"$srcbase/errors.txt"; then FAILED_PKGS="$FAILED_PKGS $pkg" -- cgit v1.2.3-2-g168b From 2f6846c451b1b4887fd87a2bc32945cce0b943a3 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 25 Sep 2009 15:07:49 -0700 Subject: sourceballs: skip based on pkgbase failures If one portion of a split package fails, assume that all packages from the same $pkgbase are going to fail as well and skip them for this run. Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 09b1df7..b7a4885 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -61,6 +61,11 @@ for repo in $repos; do continue fi + #This pkgbase has already failed. No sense in trying it again this run + if echo $FAILED_PKGS | grep "\<$pkgbase\>" >&/dev/null; then + continue + fi + #Use this file to 'whitelist' or force building some sourceballs, # skipping the license check force="" @@ -71,7 +76,7 @@ for repo in $repos; do if [ ! \( -f "$srcpath$srcpkg" -o -f "$srcpath$srcpkgbase" \) ]; then if ! $dirname/../misc-scripts/make-sourceball $force \ $pkgbase $repo $arch 2>>"$srcbase/errors.txt"; then - FAILED_PKGS="$FAILED_PKGS $pkg" + FAILED_PKGS="$FAILED_PKGS $pkgbase" fi fi done @@ -80,7 +85,7 @@ done if [ -n "$FAILED_PKGS" ]; then [ -e "$srcbase/failed.txt" ] && /bin/mv "$srcbase/failed.txt" "$srcbase/failed.txt.old" - echo "The following packages failed:" > "$srcbase/failed.txt" + echo "The following package bases failed:" > "$srcbase/failed.txt" echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u >> "$srcbase/failed.txt" fi -- cgit v1.2.3-2-g168b From ee0820aa9c71e8a76f16b11c7ad77be7cb8e1ce6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=C3=A9langer?= Date: Wed, 14 Oct 2009 15:18:09 -0400 Subject: Added new cronjob script to clean up unneeded sourceballs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This script is intended to be a weekly cron-job. It supplements the current sourceballs cleanup in make-sourceball that only removes the old sourceballs. This script removes the sourceballs of packages that were either removed completely from the repo or moved to a splitted package. It also checks the license of the packages which have a sourceball to see, in case of a license change, if the hosting of the sources is still necessary. Signed-off-by: Eric Bélanger Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs-cleanup | 61 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100755 cron-jobs/sourceballs-cleanup (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs-cleanup b/cron-jobs/sourceballs-cleanup new file mode 100755 index 0000000..c8d7b15 --- /dev/null +++ b/cron-jobs/sourceballs-cleanup @@ -0,0 +1,61 @@ +#!/bin/bash + +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" + +srcpath="$FTP_BASE/sources/" +logfile="$srcpath/cleanup.txt" + +LOCKFILE="/tmp/.sourceball-cleanup.lock" +WORKDIR="/tmp/sourceball-cleanup.$packagename.$UID" + +cleanup () { + restore_umask + rm -rf "$WORKDIR" + rm -f "$LOCKFILE" + exit 0 +} + +ctrl_c() { + echo "Interrupted" >&2 + cleanup 0 +} + +if [ -f "$LOCKFILE" ]; then + owner="$(/usr/bin/stat -c %U $LOCKFILE)" + echo "error: source tarball generation is already in progress (started by $owner)" + exit 1 +fi + +trap cleanup 0 +trap ctrl_c 2 + +/bin/touch "$LOCKFILE" + +#adjust the nice level to run at a lower priority +/usr/bin/renice +10 -p $$ > /dev/null + +set_umask +/bin/mkdir -p "$WORKDIR" +cd "$WORKDIR" + +[ -e "$logfile" ] && /bin/mv "$logfile" "$logfile.old" +echo "Orphaned sourceballs:" > "$logfile" + +for sourceball in "$srcpath"/*$SRCEXT; do + packagename=$(basename $sourceball) + packagename=${packagename%-*-*$SRCEXT} + + if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1 ; then + echo "$packagename : no longer in svn. Removing sourceball." >> "$logfile" + mv $sourceball $SOURCE_CLEANUP_DESTDIR + elif [ -z "$(ls -A "$packagename/repos")" ]; then + echo "$packagename : no longer in repos but trunk is still in svn. Removing sourceball." >> "$logfile" + mv $sourceball $SOURCE_CLEANUP_DESTDIR + elif ! source "$packagename/trunk/$BUILDSCRIPT" && chk_license ${license[@]}; then + echo "$packagename : source hosting no longer required by license. Removing sourceball." >> "$logfile" + mv $sourceball $SOURCE_CLEANUP_DESTDIR + fi +done + +cleanup 0 -- cgit v1.2.3-2-g168b From 3c1eb5b59792fbaafde12a7408201fb1e0a77df6 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 29 Oct 2009 19:10:26 -0700 Subject: Rename createFileLists to be more like the others Signed-off-by: Aaron Griffin --- cron-jobs/create-filelists | 82 ++++++++++++++++++++++++++++++++++++++++++++++ cron-jobs/createFileLists | 82 ---------------------------------------------- 2 files changed, 82 insertions(+), 82 deletions(-) create mode 100755 cron-jobs/create-filelists delete mode 100755 cron-jobs/createFileLists (limited to 'cron-jobs') diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists new file mode 100755 index 0000000..d936853 --- /dev/null +++ b/cron-jobs/create-filelists @@ -0,0 +1,82 @@ +#!/bin/bash + +reposdir=/srv/ftp/ +targetdir=$reposdir +repos="core extra testing community" + +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" + +if [ -f "/tmp/createFileList.lock" ]; then + echo "Error: createFileList allready in progress." + exit 1 +fi + +touch "/tmp/createFileList.lock" || exit 1 +TMPDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1 +CACHEDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1 + +#adjust the nice level to run at a lower priority +/usr/bin/renice +10 -p $$ > /dev/null + +case "${DBEXT}" in + *.gz) TAR_OPT="z" ;; + *.bz2) TAR_OPT="j" ;; + *.xz) TAR_OPT="J" ;; + *) echo "Unknown compression type for DBEXT=${DBEXT}" && exit 1 ;; +esac + +FILESEXT="${DBEXT//db/files}" + +cd $reposdir +for repo in $repos; do + REPO_DB_FILE=${repo}$FILESEXT + for arch in ${ARCHES[@]}; do + repodir=${repo}/os/${arch}/ + cached="no" + + # extract old file archive + if [ -f ${targetdir}${repodir}${REPO_DB_FILE} ]; then + mkdir -p ${CACHEDIR}/${repodir} + bsdtar -xf ${targetdir}${repodir}${REPO_DB_FILE} -C ${CACHEDIR}/${repodir} + cached="yes" + fi + + # create file lists + for pkg in $repodir*${PKGEXT}; do + basename=$(basename $pkg) + pkgname=$(getpkgname $basename) + tmppkgdir=${TMPDIR}/${repodir}${pkgname}/ + mkdir -p $tmppkgdir + if [ -f "${CACHEDIR}/${repodir}${pkgname}/files" ]; then +# echo "cache: $pkgname" + mv ${CACHEDIR}/${repodir}${pkgname}/files ${tmppkgdir}files + else +# echo "$repo/$arch: $pkgname" + echo '%FILES%' > ${tmppkgdir}files + bsdtar --exclude=.* -tf $pkg >> ${tmppkgdir}files + cached="no" + fi + done + + # create new file archive + if [ "$cached" == "no" ]; then + # at least one package has changed, so let's rebuild the archive +# echo "creating ${REPO_DB_FILE}/${arch}" + pkgdir=${targetdir}${repodir} + mkdir -p $pkgdir + cd ${TMPDIR}/${repodir} + [ -f "${pkgdir}${REPO_DB_FILE}.old" ] && rm "${pkgdir}${REPO_DB_FILE}.old" + [ -f "${pkgdir}${REPO_DB_FILE}" ] && mv "${pkgdir}${REPO_DB_FILE}" "${pkgdir}${REPO_DB_FILE}.old" + bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f ${pkgdir}${REPO_DB_FILE} * + fi + + cd $reposdir + done +done + +cd - >/dev/null +rm -rf $TMPDIR || exit 1 +rm -rf $CACHEDIR || exit 1 +rm -f "/tmp/createFileList.lock" || exit 1 +# echo 'done' diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists deleted file mode 100755 index d936853..0000000 --- a/cron-jobs/createFileLists +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash - -reposdir=/srv/ftp/ -targetdir=$reposdir -repos="core extra testing community" - -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" - -if [ -f "/tmp/createFileList.lock" ]; then - echo "Error: createFileList allready in progress." - exit 1 -fi - -touch "/tmp/createFileList.lock" || exit 1 -TMPDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1 -CACHEDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1 - -#adjust the nice level to run at a lower priority -/usr/bin/renice +10 -p $$ > /dev/null - -case "${DBEXT}" in - *.gz) TAR_OPT="z" ;; - *.bz2) TAR_OPT="j" ;; - *.xz) TAR_OPT="J" ;; - *) echo "Unknown compression type for DBEXT=${DBEXT}" && exit 1 ;; -esac - -FILESEXT="${DBEXT//db/files}" - -cd $reposdir -for repo in $repos; do - REPO_DB_FILE=${repo}$FILESEXT - for arch in ${ARCHES[@]}; do - repodir=${repo}/os/${arch}/ - cached="no" - - # extract old file archive - if [ -f ${targetdir}${repodir}${REPO_DB_FILE} ]; then - mkdir -p ${CACHEDIR}/${repodir} - bsdtar -xf ${targetdir}${repodir}${REPO_DB_FILE} -C ${CACHEDIR}/${repodir} - cached="yes" - fi - - # create file lists - for pkg in $repodir*${PKGEXT}; do - basename=$(basename $pkg) - pkgname=$(getpkgname $basename) - tmppkgdir=${TMPDIR}/${repodir}${pkgname}/ - mkdir -p $tmppkgdir - if [ -f "${CACHEDIR}/${repodir}${pkgname}/files" ]; then -# echo "cache: $pkgname" - mv ${CACHEDIR}/${repodir}${pkgname}/files ${tmppkgdir}files - else -# echo "$repo/$arch: $pkgname" - echo '%FILES%' > ${tmppkgdir}files - bsdtar --exclude=.* -tf $pkg >> ${tmppkgdir}files - cached="no" - fi - done - - # create new file archive - if [ "$cached" == "no" ]; then - # at least one package has changed, so let's rebuild the archive -# echo "creating ${REPO_DB_FILE}/${arch}" - pkgdir=${targetdir}${repodir} - mkdir -p $pkgdir - cd ${TMPDIR}/${repodir} - [ -f "${pkgdir}${REPO_DB_FILE}.old" ] && rm "${pkgdir}${REPO_DB_FILE}.old" - [ -f "${pkgdir}${REPO_DB_FILE}" ] && mv "${pkgdir}${REPO_DB_FILE}" "${pkgdir}${REPO_DB_FILE}.old" - bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f ${pkgdir}${REPO_DB_FILE} * - fi - - cd $reposdir - done -done - -cd - >/dev/null -rm -rf $TMPDIR || exit 1 -rm -rf $CACHEDIR || exit 1 -rm -f "/tmp/createFileList.lock" || exit 1 -# echo 'done' -- cgit v1.2.3-2-g168b From 38c7241ad2528b29a65a0b81b980e76d1046e94e Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 29 Oct 2009 19:18:41 -0700 Subject: Cleanup create-filelists * Quote absolutely everything * Move directory slashes around, for readability * Move a cd to the top of the loop Signed-off-by: Aaron Griffin --- cron-jobs/create-filelists | 55 +++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 28 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index d936853..b49f722 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -1,7 +1,7 @@ #!/bin/bash -reposdir=/srv/ftp/ -targetdir=$reposdir +reposdir="/srv/ftp" +targetdir="/srv/ftp" repos="core extra testing community" . "$(dirname $0)/../db-functions" @@ -13,8 +13,8 @@ if [ -f "/tmp/createFileList.lock" ]; then fi touch "/tmp/createFileList.lock" || exit 1 -TMPDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1 -CACHEDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1 +TMPDIR="$(mktemp -d /tmp/createFileList.XXXXXX)" || exit 1 +CACHEDIR="$(mktemp -d /tmp/createFileList.XXXXXX)" || exit 1 #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null @@ -28,33 +28,34 @@ esac FILESEXT="${DBEXT//db/files}" -cd $reposdir for repo in $repos; do - REPO_DB_FILE=${repo}$FILESEXT + REPO_DB_FILE="${repo}$FILESEXT" for arch in ${ARCHES[@]}; do - repodir=${repo}/os/${arch}/ + cd "$reposdir" + + repodir="${repo}/os/${arch}" cached="no" # extract old file archive - if [ -f ${targetdir}${repodir}${REPO_DB_FILE} ]; then - mkdir -p ${CACHEDIR}/${repodir} - bsdtar -xf ${targetdir}${repodir}${REPO_DB_FILE} -C ${CACHEDIR}/${repodir} + if [ -f "${targetdir}/${repodir}/${REPO_DB_FILE}" ]; then + mkdir -p "${CACHEDIR}/${repodir}" + bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${CACHEDIR}/${repodir}" cached="yes" fi # create file lists for pkg in $repodir*${PKGEXT}; do - basename=$(basename $pkg) - pkgname=$(getpkgname $basename) - tmppkgdir=${TMPDIR}/${repodir}${pkgname}/ - mkdir -p $tmppkgdir - if [ -f "${CACHEDIR}/${repodir}${pkgname}/files" ]; then + basename="$(basename "$pkg")" + pkgname="$(getpkgname "$basename")" + tmppkgdir="${TMPDIR}/${repodir}/${pkgname}" + mkdir -p "$tmppkgdir" + if [ -f "${CACHEDIR}/${repodir}/${pkgname}/files" ]; then # echo "cache: $pkgname" - mv ${CACHEDIR}/${repodir}${pkgname}/files ${tmppkgdir}files + mv "${CACHEDIR}/${repodir}$/{pkgname}/files" "${tmppkgdir}/files" else # echo "$repo/$arch: $pkgname" - echo '%FILES%' > ${tmppkgdir}files - bsdtar --exclude=.* -tf $pkg >> ${tmppkgdir}files + echo '%FILES%' > "${tmppkgdir}/files" + bsdtar --exclude=.* -tf "$pkg" >> "${tmppkgdir}/files" cached="no" fi done @@ -63,20 +64,18 @@ for repo in $repos; do if [ "$cached" == "no" ]; then # at least one package has changed, so let's rebuild the archive # echo "creating ${REPO_DB_FILE}/${arch}" - pkgdir=${targetdir}${repodir} - mkdir -p $pkgdir - cd ${TMPDIR}/${repodir} - [ -f "${pkgdir}${REPO_DB_FILE}.old" ] && rm "${pkgdir}${REPO_DB_FILE}.old" - [ -f "${pkgdir}${REPO_DB_FILE}" ] && mv "${pkgdir}${REPO_DB_FILE}" "${pkgdir}${REPO_DB_FILE}.old" - bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f ${pkgdir}${REPO_DB_FILE} * + pkgdir="${targetdir}/${repodir}" + mkdir -p "$pkgdir" + cd "${TMPDIR}/${repodir}" + [ -f "${pkgdir}/${REPO_DB_FILE}.old" ] && rm "${pkgdir}/${REPO_DB_FILE}.old" + [ -f "${pkgdir}/${REPO_DB_FILE}" ] && mv "${pkgdir}/${REPO_DB_FILE}" "${pkgdir}/${REPO_DB_FILE}.old" + bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f "${pkgdir}/${REPO_DB_FILE}" * fi - - cd $reposdir done done cd - >/dev/null -rm -rf $TMPDIR || exit 1 -rm -rf $CACHEDIR || exit 1 +rm -rf "$TMPDIR" || exit 1 +rm -rf "$CACHEDIR" || exit 1 rm -f "/tmp/createFileList.lock" || exit 1 # echo 'done' -- cgit v1.2.3-2-g168b From 78c6a7916836fd10fb232d617b6eca3ae5c48089 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 29 Oct 2009 19:19:44 -0700 Subject: create-filelists: Call getpkgname for real Signed-off-by: Aaron Griffin --- cron-jobs/create-filelists | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index b49f722..35325cf 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -45,8 +45,7 @@ for repo in $repos; do # create file lists for pkg in $repodir*${PKGEXT}; do - basename="$(basename "$pkg")" - pkgname="$(getpkgname "$basename")" + pkgname="$(getpkgname "$pkg")" tmppkgdir="${TMPDIR}/${repodir}/${pkgname}" mkdir -p "$tmppkgdir" if [ -f "${CACHEDIR}/${repodir}/${pkgname}/files" ]; then -- cgit v1.2.3-2-g168b From 1857fca2d204011852cf2309b66381b2fb4f6535 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20B=C3=A4chler?= Date: Sat, 31 Oct 2009 19:06:33 +0100 Subject: Fix a typo introduced in 38c7241ad2528b29a65a0b81b980e76d1046e94e --- cron-jobs/create-filelists | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index 35325cf..727fe58 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -50,7 +50,7 @@ for repo in $repos; do mkdir -p "$tmppkgdir" if [ -f "${CACHEDIR}/${repodir}/${pkgname}/files" ]; then # echo "cache: $pkgname" - mv "${CACHEDIR}/${repodir}$/{pkgname}/files" "${tmppkgdir}/files" + mv "${CACHEDIR}/${repodir}/${pkgname}/files" "${tmppkgdir}/files" else # echo "$repo/$arch: $pkgname" echo '%FILES%' > "${tmppkgdir}/files" -- cgit v1.2.3-2-g168b From 782e8d67158115e8040215bcfef7e8ba54c6d04e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20B=C3=A4chler?= Date: Sat, 31 Oct 2009 19:17:39 +0100 Subject: create-filelists: add community-testing --- cron-jobs/create-filelists | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index 727fe58..840535e 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -2,7 +2,7 @@ reposdir="/srv/ftp" targetdir="/srv/ftp" -repos="core extra testing community" +repos="core extra testing community community-testing" . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" -- cgit v1.2.3-2-g168b From c54f6c91dca7d84480f22aee60dda5861c73712e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20B=C3=A4chler?= Date: Sat, 31 Oct 2009 19:28:38 +0100 Subject: create-filelists: fix another typo --- cron-jobs/create-filelists | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index 840535e..2a15e6a 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -44,7 +44,7 @@ for repo in $repos; do fi # create file lists - for pkg in $repodir*${PKGEXT}; do + for pkg in $repodir/*${PKGEXT}; do pkgname="$(getpkgname "$pkg")" tmppkgdir="${TMPDIR}/${repodir}/${pkgname}" mkdir -p "$tmppkgdir" -- cgit v1.2.3-2-g168b From 27e8f1071c18b2f0cbf4deb84a1222db64a3d5dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20B=C3=A4chler?= Date: Sat, 31 Oct 2009 20:07:17 +0100 Subject: create-filelists: Add pkgver-pkgrel to directory name In the past, pkgname-pkgver-pkgrel was extracted from the filename and sadly named "pkgname". Restore this behaviour --- cron-jobs/create-filelists | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index 2a15e6a..c9d7db9 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -46,11 +46,12 @@ for repo in $repos; do # create file lists for pkg in $repodir/*${PKGEXT}; do pkgname="$(getpkgname "$pkg")" - tmppkgdir="${TMPDIR}/${repodir}/${pkgname}" + pkgver="$(getpkgver "$pkg")" + tmppkgdir="${TMPDIR}/${repodir}/${pkgname}-${pkgver}" mkdir -p "$tmppkgdir" - if [ -f "${CACHEDIR}/${repodir}/${pkgname}/files" ]; then + if [ -f "${CACHEDIR}/${repodir}/${pkgname}-${pkgver}/files" ]; then # echo "cache: $pkgname" - mv "${CACHEDIR}/${repodir}/${pkgname}/files" "${tmppkgdir}/files" + mv "${CACHEDIR}/${repodir}/${pkgname}-${pkgver}/files" "${tmppkgdir}/files" else # echo "$repo/$arch: $pkgname" echo '%FILES%' > "${tmppkgdir}/files" -- cgit v1.2.3-2-g168b From 1d6421c7a12e9a07d00ec4de999ae95f7dd0874e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 27 Feb 2010 10:47:52 -0600 Subject: create-filelists: general cleanups * Specify lock name once * Use new script name everywhere * Clean up tabs/spaces and add a modeline. This isn't necessarily the one we wanted to standardize on, but I picked the one the entire file is written to at the moment. Signed-off-by: Dan McGee --- cron-jobs/create-filelists | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index c9d7db9..e90da00 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -3,27 +3,28 @@ reposdir="/srv/ftp" targetdir="/srv/ftp" repos="core extra testing community community-testing" +lock="/tmp/create-filelists.lock" . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" -if [ -f "/tmp/createFileList.lock" ]; then - echo "Error: createFileList allready in progress." +if [ -f "$lock" ]; then + echo "Error: create-filelists already in progress." exit 1 fi -touch "/tmp/createFileList.lock" || exit 1 -TMPDIR="$(mktemp -d /tmp/createFileList.XXXXXX)" || exit 1 -CACHEDIR="$(mktemp -d /tmp/createFileList.XXXXXX)" || exit 1 +touch "$lock" || exit 1 +TMPDIR="$(mktemp -d /tmp/create-filelists.XXXXXX)" || exit 1 +CACHEDIR="$(mktemp -d /tmp/create-filelists.XXXXXX)" || exit 1 #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null case "${DBEXT}" in - *.gz) TAR_OPT="z" ;; - *.bz2) TAR_OPT="j" ;; - *.xz) TAR_OPT="J" ;; - *) echo "Unknown compression type for DBEXT=${DBEXT}" && exit 1 ;; + *.gz) TAR_OPT="z" ;; + *.bz2) TAR_OPT="j" ;; + *.xz) TAR_OPT="J" ;; + *) echo "Unknown compression type for DBEXT=${DBEXT}" && exit 1 ;; esac FILESEXT="${DBEXT//db/files}" @@ -77,5 +78,7 @@ done cd - >/dev/null rm -rf "$TMPDIR" || exit 1 rm -rf "$CACHEDIR" || exit 1 -rm -f "/tmp/createFileList.lock" || exit 1 +rm -f "$lock" || exit 1 # echo 'done' + +# vim: set ts=4 sw=4 et ft=sh: -- cgit v1.2.3-2-g168b From 6587ebadf6b3a0d8e58be1ab7169a4c2f88ed57c Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 27 Feb 2010 10:58:22 -0600 Subject: create-filelists: s/REPO_DB_FILE/FILES_DB_FILE/g This will set up changes soon to come where we actually use the real repos DB file so I don't want variable name confusion. Signed-off-by: Dan McGee --- cron-jobs/create-filelists | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index e90da00..a0b6a57 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -30,7 +30,7 @@ esac FILESEXT="${DBEXT//db/files}" for repo in $repos; do - REPO_DB_FILE="${repo}$FILESEXT" + FILES_DB_FILE="${repo}$FILESEXT" for arch in ${ARCHES[@]}; do cd "$reposdir" @@ -38,9 +38,9 @@ for repo in $repos; do cached="no" # extract old file archive - if [ -f "${targetdir}/${repodir}/${REPO_DB_FILE}" ]; then + if [ -f "${targetdir}/${repodir}/${FILES_DB_FILE}" ]; then mkdir -p "${CACHEDIR}/${repodir}" - bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${CACHEDIR}/${repodir}" + bsdtar -xf "${targetdir}/${repodir}/${FILES_DB_FILE}" -C "${CACHEDIR}/${repodir}" cached="yes" fi @@ -64,13 +64,13 @@ for repo in $repos; do # create new file archive if [ "$cached" == "no" ]; then # at least one package has changed, so let's rebuild the archive -# echo "creating ${REPO_DB_FILE}/${arch}" +# echo "creating ${FILES_DB_FILE}/${arch}" pkgdir="${targetdir}/${repodir}" mkdir -p "$pkgdir" cd "${TMPDIR}/${repodir}" - [ -f "${pkgdir}/${REPO_DB_FILE}.old" ] && rm "${pkgdir}/${REPO_DB_FILE}.old" - [ -f "${pkgdir}/${REPO_DB_FILE}" ] && mv "${pkgdir}/${REPO_DB_FILE}" "${pkgdir}/${REPO_DB_FILE}.old" - bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f "${pkgdir}/${REPO_DB_FILE}" * + [ -f "${pkgdir}/${FILES_DB_FILE}.old" ] && rm "${pkgdir}/${FILES_DB_FILE}.old" + [ -f "${pkgdir}/${FILES_DB_FILE}" ] && mv "${pkgdir}/${FILES_DB_FILE}" "${pkgdir}/${FILES_DB_FILE}.old" + bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f "${pkgdir}/${FILES_DB_FILE}" * fi done done -- cgit v1.2.3-2-g168b From ff1530def072daf95f077ec0f8a4d984da4304d6 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 27 Feb 2010 11:27:13 -0600 Subject: create-filelists: rework the package loop completely Instead of wasting time extracting .PKGINFO twice from every single package in the repos, use the package DB to eliminate most of the heavy lifting. This way we only need to worry about looking at the packages that actually have changed since the last time we built the package database. This should give a noticeable performance increase to this job in addition to reducing IO load and unnecessary reading of every package file. Signed-off-by: Dan McGee --- cron-jobs/create-filelists | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index a0b6a57..6091bf4 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -14,8 +14,12 @@ if [ -f "$lock" ]; then fi touch "$lock" || exit 1 -TMPDIR="$(mktemp -d /tmp/create-filelists.XXXXXX)" || exit 1 -CACHEDIR="$(mktemp -d /tmp/create-filelists.XXXXXX)" || exit 1 +# location where the package DB is extracted so we know what to include +DBDIR="$(mktemp -d /tmp/create-filelists.dbdir.XXXXXX)" || exit 1 +# location where the old files DB is extracted to save us some work +CACHEDIR="$(mktemp -d /tmp/create-filelists.cachedir.XXXXXX)" || exit 1 +# location where the new files DB is built up and eventually zipped +TMPDIR="$(mktemp -d /tmp/create-filelists.tmpdir.XXXXXX)" || exit 1 #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null @@ -30,33 +34,45 @@ esac FILESEXT="${DBEXT//db/files}" for repo in $repos; do + REPO_DB_FILE="${repo}$DBEXT" FILES_DB_FILE="${repo}$FILESEXT" for arch in ${ARCHES[@]}; do +# echo "Running for architecture $arch, repo $repo" cd "$reposdir" repodir="${repo}/os/${arch}" cached="no" + # extract package db archive + if [ -f "${targetdir}/${repodir}/${REPO_DB_FILE}" ]; then + mkdir -p "${DBDIR}/${repodir}" +# echo "extracting $REPO_DB_FILE" + bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${DBDIR}/${repodir}" + else + echo "Fail! Does the repo $repo with arch $arch even exist?" + continue + fi + # extract old file archive if [ -f "${targetdir}/${repodir}/${FILES_DB_FILE}" ]; then mkdir -p "${CACHEDIR}/${repodir}" +# echo "extracting $FILES_DB_FILE" bsdtar -xf "${targetdir}/${repodir}/${FILES_DB_FILE}" -C "${CACHEDIR}/${repodir}" cached="yes" fi # create file lists - for pkg in $repodir/*${PKGEXT}; do - pkgname="$(getpkgname "$pkg")" - pkgver="$(getpkgver "$pkg")" - tmppkgdir="${TMPDIR}/${repodir}/${pkgname}-${pkgver}" + for pkg in $(ls ${DBDIR}/${repodir}); do + tmppkgdir="${TMPDIR}/${repodir}/${pkg}" mkdir -p "$tmppkgdir" - if [ -f "${CACHEDIR}/${repodir}/${pkgname}-${pkgver}/files" ]; then -# echo "cache: $pkgname" - mv "${CACHEDIR}/${repodir}/${pkgname}-${pkgver}/files" "${tmppkgdir}/files" + if [ -f "${CACHEDIR}/${repodir}/${pkg}/files" ]; then +# echo "cache: $pkg" + mv "${CACHEDIR}/${repodir}/${pkg}/files" "${tmppkgdir}/files" else -# echo "$repo/$arch: $pkgname" +# echo "not cache: $repo/$arch: $pkg" + filename=$(grep -A1 '^%FILENAME%$' "${DBDIR}/${repodir}/${pkg}/desc" | tail -n1) echo '%FILES%' > "${tmppkgdir}/files" - bsdtar --exclude=.* -tf "$pkg" >> "${tmppkgdir}/files" + bsdtar --exclude=.* -tf "$repodir/$filename" >> "${tmppkgdir}/files" cached="no" fi done @@ -76,8 +92,7 @@ for repo in $repos; do done cd - >/dev/null -rm -rf "$TMPDIR" || exit 1 -rm -rf "$CACHEDIR" || exit 1 +rm -rf "$TMPDIR" "$CACHEDIR" "$DBDIR" rm -f "$lock" || exit 1 # echo 'done' -- cgit v1.2.3-2-g168b From ffa88b335b2f756925b388ecbd6681bf3aa58579 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 27 Feb 2010 11:52:56 -0600 Subject: create-filelists: include desc/depends entries Make the files DB include everything the original packages DB includes instead of just being 'files' entries. This will allow tools to do more with these generated files and they can be used as a drop-in replacement for a regular package database. Signed-off-by: Dan McGee --- cron-jobs/create-filelists | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index 6091bf4..84867d8 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -63,14 +63,18 @@ for repo in $repos; do # create file lists for pkg in $(ls ${DBDIR}/${repodir}); do + dbpkgdir="${DBDIR}/${repodir}/${pkg}" + cachepkgdir="${CACHEDIR}/${repodir}/${pkg}" tmppkgdir="${TMPDIR}/${repodir}/${pkg}" mkdir -p "$tmppkgdir" - if [ -f "${CACHEDIR}/${repodir}/${pkg}/files" ]; then + ln "${dbpkgdir}/desc" "${tmppkgdir}/desc" + ln "${dbpkgdir}/depends" "${tmppkgdir}/depends" + if [ -f "${cachepkgdir}/files" ]; then # echo "cache: $pkg" - mv "${CACHEDIR}/${repodir}/${pkg}/files" "${tmppkgdir}/files" + ln "${cachepkgdir}/files" "${tmppkgdir}/files" else # echo "not cache: $repo/$arch: $pkg" - filename=$(grep -A1 '^%FILENAME%$' "${DBDIR}/${repodir}/${pkg}/desc" | tail -n1) + filename=$(grep -A1 '^%FILENAME%$' "${dbpkgdir}/desc" | tail -n1) echo '%FILES%' > "${tmppkgdir}/files" bsdtar --exclude=.* -tf "$repodir/$filename" >> "${tmppkgdir}/files" cached="no" -- cgit v1.2.3-2-g168b From e7fe897a91e6376a8724c226f8e078251620a173 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=C3=A9langer?= Date: Mon, 8 Feb 2010 10:13:42 -0500 Subject: sourceballs: Make cleanup more efficient MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moved all cleanup related code in sourceballs-cleanup script and moved, now common, functions in db-functions. The cleanup script is now ran after all the new sourceballs have been fetched. Signed-off-by: Eric Bélanger --- cron-jobs/sourceballs | 2 ++ 1 file changed, 2 insertions(+) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index b7a4885..ccb00ad 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -89,4 +89,6 @@ if [ -n "$FAILED_PKGS" ]; then echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u >> "$srcbase/failed.txt" fi +$dirname/../misc-scripts/sourceballs-cleanup + cleanup -- cgit v1.2.3-2-g168b From 7bc888905280e1b59febdc4bad475b726225b95d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=C3=A9langer?= Date: Mon, 8 Feb 2010 10:13:43 -0500 Subject: moving sourceballs-cleanup --- cron-jobs/sourceballs-cleanup | 61 ------------------------------------------- 1 file changed, 61 deletions(-) delete mode 100755 cron-jobs/sourceballs-cleanup (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs-cleanup b/cron-jobs/sourceballs-cleanup deleted file mode 100755 index c8d7b15..0000000 --- a/cron-jobs/sourceballs-cleanup +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" - -srcpath="$FTP_BASE/sources/" -logfile="$srcpath/cleanup.txt" - -LOCKFILE="/tmp/.sourceball-cleanup.lock" -WORKDIR="/tmp/sourceball-cleanup.$packagename.$UID" - -cleanup () { - restore_umask - rm -rf "$WORKDIR" - rm -f "$LOCKFILE" - exit 0 -} - -ctrl_c() { - echo "Interrupted" >&2 - cleanup 0 -} - -if [ -f "$LOCKFILE" ]; then - owner="$(/usr/bin/stat -c %U $LOCKFILE)" - echo "error: source tarball generation is already in progress (started by $owner)" - exit 1 -fi - -trap cleanup 0 -trap ctrl_c 2 - -/bin/touch "$LOCKFILE" - -#adjust the nice level to run at a lower priority -/usr/bin/renice +10 -p $$ > /dev/null - -set_umask -/bin/mkdir -p "$WORKDIR" -cd "$WORKDIR" - -[ -e "$logfile" ] && /bin/mv "$logfile" "$logfile.old" -echo "Orphaned sourceballs:" > "$logfile" - -for sourceball in "$srcpath"/*$SRCEXT; do - packagename=$(basename $sourceball) - packagename=${packagename%-*-*$SRCEXT} - - if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1 ; then - echo "$packagename : no longer in svn. Removing sourceball." >> "$logfile" - mv $sourceball $SOURCE_CLEANUP_DESTDIR - elif [ -z "$(ls -A "$packagename/repos")" ]; then - echo "$packagename : no longer in repos but trunk is still in svn. Removing sourceball." >> "$logfile" - mv $sourceball $SOURCE_CLEANUP_DESTDIR - elif ! source "$packagename/trunk/$BUILDSCRIPT" && chk_license ${license[@]}; then - echo "$packagename : source hosting no longer required by license. Removing sourceball." >> "$logfile" - mv $sourceball $SOURCE_CLEANUP_DESTDIR - fi -done - -cleanup 0 -- cgit v1.2.3-2-g168b From 57987ae42fb58c58e044bfd7d3486010b32bb313 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Wed, 31 Mar 2010 23:09:57 +0200 Subject: don't publish *.old files --- cron-jobs/create-filelists | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index 84867d8..a25fd1a 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -88,8 +88,7 @@ for repo in $repos; do pkgdir="${targetdir}/${repodir}" mkdir -p "$pkgdir" cd "${TMPDIR}/${repodir}" - [ -f "${pkgdir}/${FILES_DB_FILE}.old" ] && rm "${pkgdir}/${FILES_DB_FILE}.old" - [ -f "${pkgdir}/${FILES_DB_FILE}" ] && mv "${pkgdir}/${FILES_DB_FILE}" "${pkgdir}/${FILES_DB_FILE}.old" + [ -f "${pkgdir}/${FILES_DB_FILE}" ] && rm "${pkgdir}/${FILES_DB_FILE}" bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f "${pkgdir}/${FILES_DB_FILE}" * fi done -- cgit v1.2.3-2-g168b From e284d06d6cf63665cbadcb65e3a09bbd18027bd1 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 15 Feb 2010 15:44:46 -0800 Subject: Change ftpdir-cleanup scripts for package pooling * The per repo script now simply removes files as they should all be symlinks at this time * The cron job script will do the extra job of checking the global pool. If there are no more symlinks, move the pooled package Signed-off-by: Aaron Griffin --- cron-jobs/ftpdir-cleanup | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index d66138c..1ccc654 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -31,7 +31,40 @@ trap ctrl_c 2 /usr/bin/renice +10 -p $$ > /dev/null for repo in $repos; do - $(dirname $0)/../misc-scripts/ftpdir-cleanup $repo $CLEANUP_DESTDIR + $(dirname $0)/../misc-scripts/ftpdir-cleanup $repo done +to_cleanup="" +for _arch in ${ARCHES[@]}; do + poolpath="$FTP_BASE/packages/os/$_arch/" + cd $poolpath + for pkg in *$PKGEXT; do + [ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" + LINKS="$(/bin/ls $FTP_BASE/*/os/$_arch/$pkg 2>/dev/null)" + if [ -n "$LINKS" ]; then + found=0 + for lnk in $LINKS; do + if [ -h "$lnk" ]; then + found=1 + break + fi + done + # No links found, clean it up + if [ $found -eq 0 ]; then + to_cleanup="$to_cleanup $poolpath/$pkg" + fi + fi + done +done + +if [ -n "$to_cleanup" ]; then + echo " The following packages are no longer in any repo" + echo " They will be moved to $CLEANUP_DESTDIR" + for f in $to_cleanup; do + echo " $(basename "$f")" + done + echo "" + mv "$f" "$CLEANUP_DESTDIR" +fi + cleanup -- cgit v1.2.3-2-g168b From 709b4a514e24620145756a312ef9aab0662a3d8f Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 28 Apr 2010 11:55:04 -0700 Subject: ftpdir-cleanup: Fix cleanup of package pool --- cron-jobs/ftpdir-cleanup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 1ccc654..2496ced 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -64,7 +64,7 @@ if [ -n "$to_cleanup" ]; then echo " $(basename "$f")" done echo "" - mv "$f" "$CLEANUP_DESTDIR" + mv $to_cleanup "$CLEANUP_DESTDIR" fi cleanup -- cgit v1.2.3-2-g168b From 22c5f235fa25ea3327d95a28c6dc8ae399d6cb9a Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 28 Apr 2010 12:00:48 -0700 Subject: ftpdir-cleanup: use 'find' for symlink detection This gets rid of a bash loop for link detection --- cron-jobs/ftpdir-cleanup | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 2496ced..977b384 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -40,19 +40,9 @@ for _arch in ${ARCHES[@]}; do cd $poolpath for pkg in *$PKGEXT; do [ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" - LINKS="$(/bin/ls $FTP_BASE/*/os/$_arch/$pkg 2>/dev/null)" - if [ -n "$LINKS" ]; then - found=0 - for lnk in $LINKS; do - if [ -h "$lnk" ]; then - found=1 - break - fi - done - # No links found, clean it up - if [ $found -eq 0 ]; then - to_cleanup="$to_cleanup $poolpath/$pkg" - fi + LINKS="$(/usr/bin/find $FTP_BASE -type l -name "$pkg" 2>/dev/null)" + if [ -z "$LINKS" ]; then + to_cleanup="$to_cleanup $poolpath/$pkg" fi done done -- cgit v1.2.3-2-g168b From ee4074de910df5c637691ace06ec5a9d475fb4a3 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 3 May 2010 09:17:04 -0700 Subject: Add PKGPOOL_DIR config variable This is a relative var, descendent from FTP_BASE. We do this because we also need a relative path to the package pool dir --- cron-jobs/ftpdir-cleanup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 977b384..a278341 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -36,7 +36,7 @@ done to_cleanup="" for _arch in ${ARCHES[@]}; do - poolpath="$FTP_BASE/packages/os/$_arch/" + poolpath="$FTP_BASE/$PKGPOOL_DIR/os/$_arch/" cd $poolpath for pkg in *$PKGEXT; do [ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" -- cgit v1.2.3-2-g168b From 5034b4e644c55631dfd5c88b491873389fc52e50 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 20 Jun 2010 17:29:40 +0200 Subject: Replace PKGPOOL_DIR by a function call The package pool dir depends on the host; so we solve this similar to get_repos_for_host() Note: There is no "os" subdir for the package pool. --- cron-jobs/ftpdir-cleanup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index a278341..2e23440 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -36,7 +36,7 @@ done to_cleanup="" for _arch in ${ARCHES[@]}; do - poolpath="$FTP_BASE/$PKGPOOL_DIR/os/$_arch/" + poolpath="$FTP_BASE/$(get_pkgpool_for_host)/$_arch/" cd $poolpath for pkg in *$PKGEXT; do [ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" -- cgit v1.2.3-2-g168b From 0f705986ab0898a1cd23c3e6d25a118cbd804648 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 20 Jun 2010 18:16:07 +0200 Subject: ftpdir-cleanup: only search for linked packages in actual repos --- cron-jobs/ftpdir-cleanup | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 2e23440..66733f0 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -30,8 +30,10 @@ trap ctrl_c 2 #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null +repopaths='' for repo in $repos; do $(dirname $0)/../misc-scripts/ftpdir-cleanup $repo + repopaths="${repopaths} ${FTP_BASE}/${repo}/os/" done to_cleanup="" @@ -40,7 +42,7 @@ for _arch in ${ARCHES[@]}; do cd $poolpath for pkg in *$PKGEXT; do [ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" - LINKS="$(/usr/bin/find $FTP_BASE -type l -name "$pkg" 2>/dev/null)" + LINKS="$(/usr/bin/find $repopaths -type l -name "$pkg" 2>/dev/null)" if [ -z "$LINKS" ]; then to_cleanup="$to_cleanup $poolpath/$pkg" fi -- cgit v1.2.3-2-g168b From f6004585ef7ceff8e0119363f297104ac7900e09 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 20 Jun 2010 18:45:55 +0200 Subject: adjust permissions for package pool --- cron-jobs/adjust-permissions | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index 0f87075..c92cb9e 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -21,19 +21,29 @@ get_dir_owner() { echo "root:tusers" ;; community-testing) echo "root:tusers" ;; + packages/arch) + echo "ftp:ftp-extra" ;; + packages/community) + echo "root:tusers" ;; esac } #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null -cd /srv/ftp +cd $FTP_BASE for d in $(get_repos_for_host); do owner="$(get_dir_owner $d)" /bin/chown -R $owner $d/os/{any,i686,x86_64} /bin/chmod -R g+w $d/os/{any,i686,x86_64} done -/bin/chmod 555 /srv/ftp +for p in $(get_pkgpool_for_host); do + owner="$(get_dir_owner $p)" + /bin/chown -R $owner $p/{any,i686,x86_64} + /bin/chmod -R g+w $p/{any,i686,x86_64} +done + +/bin/chmod 555 $FTP_BASE rm -f /tmp/.ftpmaint.lck -- cgit v1.2.3-2-g168b From e2cc14311b1bfad0c30cef395a1a033cdef030c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=C3=A9langer?= Date: Sun, 27 Jun 2010 23:40:38 -0400 Subject: sourceballs: Added community and community-testing support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Eric Bélanger Signed-off-by: Pierre Schmitz --- cron-jobs/sourceballs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index ccb00ad..93cb842 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -2,7 +2,7 @@ ftpbase="/srv/ftp" srcbase="/srv/ftp/sources" -repos="core extra testing" #community" +repos="core extra testing community community-testing" . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" -- cgit v1.2.3-2-g168b From 3e08da59e3ca3849b753318b791aed8f1f8d8fb4 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Aug 2010 01:26:52 +0200 Subject: fix cleaning of any packages and ignore empty repos --- cron-jobs/ftpdir-cleanup | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 66733f0..546afd9 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -37,9 +37,9 @@ for repo in $repos; do done to_cleanup="" -for _arch in ${ARCHES[@]}; do +for _arch in any ${ARCHES[@]}; do poolpath="$FTP_BASE/$(get_pkgpool_for_host)/$_arch/" - cd $poolpath + pushd $poolpath >/dev/null for pkg in *$PKGEXT; do [ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" LINKS="$(/usr/bin/find $repopaths -type l -name "$pkg" 2>/dev/null)" @@ -47,6 +47,7 @@ for _arch in ${ARCHES[@]}; do to_cleanup="$to_cleanup $poolpath/$pkg" fi done + popd >/dev/null done if [ -n "$to_cleanup" ]; then -- cgit v1.2.3-2-g168b From 9eb1cd7b9403533c4b60ecfbbbf00a08c211059a Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Aug 2010 15:03:27 +0200 Subject: Move common function to db-functions db-functions now sets an individual $WORKDIR and implements trap functinos that remove locks on exit or error. There are new functions to lock and unlock the running script. misc-scripts/ftpdir-cleanup was renamed to ftpdir-cleanup-repo as the cron-job had the same name. Script names have to be unique when using db-functions. --- cron-jobs/ftpdir-cleanup | 26 +++----------------------- 1 file changed, 3 insertions(+), 23 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 546afd9..f5e401a 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -5,34 +5,14 @@ repos="$(get_repos_for_host)" -LOCKFILE="/tmp/.ftpdircleanup.lock" - -cleanup () { - rm -f "$LOCKFILE" - exit 0 -} - -ctrl_c() { - cleanup -} - -if [ -f "$LOCKFILE" ]; then - owner="$(/usr/bin/stat -c %U $LOCKFILE)" - echo "error: ftp cleanup is already in progress (started by $owner)" - exit 1 -fi - -trap cleanup 0 -trap ctrl_c 2 - -/bin/touch "$LOCKFILE" +script_lock #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null repopaths='' for repo in $repos; do - $(dirname $0)/../misc-scripts/ftpdir-cleanup $repo + $(dirname $0)/../misc-scripts/ftpdir-cleanup-repo $repo repopaths="${repopaths} ${FTP_BASE}/${repo}/os/" done @@ -60,4 +40,4 @@ if [ -n "$to_cleanup" ]; then mv $to_cleanup "$CLEANUP_DESTDIR" fi -cleanup +script_unlock -- cgit v1.2.3-2-g168b From 6326b0e86214a37d9f9be62bc84a79b015ff3633 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Aug 2010 15:11:14 +0200 Subject: Cleanup cron-jobs/adjust-permissions --- cron-jobs/adjust-permissions | 57 ++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 31 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index c92cb9e..c21af0c 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -1,49 +1,44 @@ #!/bin/bash -if [ -f /tmp/.ftpmaint.lck ]; then - exit 0 -fi - -/bin/touch /tmp/.ftpmaint.lck +script_lock . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" get_dir_owner() { - case $1 in - core) - echo "ftp:ftp-arch" ;; - extra) - echo "ftp:ftp-extra" ;; - testing) - echo "ftp:ftp-extra" ;; - community) - echo "root:tusers" ;; - community-testing) - echo "root:tusers" ;; - packages/arch) - echo "ftp:ftp-extra" ;; - packages/community) - echo "root:tusers" ;; - esac + case $1 in + core) + echo "ftp:ftp-arch" ;; + extra) + echo "ftp:ftp-extra" ;; + testing) + echo "ftp:ftp-extra" ;; + community) + echo "root:tusers" ;; + community-testing) + echo "root:tusers" ;; + packages/arch) + echo "ftp:ftp-extra" ;; + packages/community) + echo "root:tusers" ;; + esac } #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null -cd $FTP_BASE +pushd $FTP_BASE >/dev/null for d in $(get_repos_for_host); do - owner="$(get_dir_owner $d)" - /bin/chown -R $owner $d/os/{any,i686,x86_64} - /bin/chmod -R g+w $d/os/{any,i686,x86_64} + owner="$(get_dir_owner $d)" + /bin/chown -R $owner $d/os/{any,i686,x86_64} + /bin/chmod -R g+w $d/os/{any,i686,x86_64} done - for p in $(get_pkgpool_for_host); do - owner="$(get_dir_owner $p)" - /bin/chown -R $owner $p/{any,i686,x86_64} - /bin/chmod -R g+w $p/{any,i686,x86_64} + owner="$(get_dir_owner $p)" + /bin/chown -R $owner $p/{any,i686,x86_64} + /bin/chmod -R g+w $p/{any,i686,x86_64} done - /bin/chmod 555 $FTP_BASE +popd >/dev/null -rm -f /tmp/.ftpmaint.lck +script_unlock -- cgit v1.2.3-2-g168b From fdca04b3f5d8e27af4829416fee3843b0fe01e36 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Aug 2010 16:43:16 +0200 Subject: Use db-functions in create-filelists and sourceballs --- cron-jobs/create-filelists | 50 ++++++++++++++++++---------------------------- cron-jobs/sourceballs | 33 ++++++------------------------ 2 files changed, 25 insertions(+), 58 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index a25fd1a..8c354fc 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -1,44 +1,38 @@ #!/bin/bash -reposdir="/srv/ftp" -targetdir="/srv/ftp" -repos="core extra testing community community-testing" -lock="/tmp/create-filelists.lock" - . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" -if [ -f "$lock" ]; then - echo "Error: create-filelists already in progress." - exit 1 -fi +reposdir=${FTP_BASE} +targetdir=${FTP_BASE} +repos=(core extra testing community community-testing) + +script_lock -touch "$lock" || exit 1 # location where the package DB is extracted so we know what to include -DBDIR="$(mktemp -d /tmp/create-filelists.dbdir.XXXXXX)" || exit 1 +DBDIR="$(mktemp -d ${WORKDIR}/create-filelists.dbdir.XXXXXX)" || exit 1 # location where the old files DB is extracted to save us some work -CACHEDIR="$(mktemp -d /tmp/create-filelists.cachedir.XXXXXX)" || exit 1 +CACHEDIR="$(mktemp -d ${WORKDIR}/create-filelists.cachedir.XXXXXX)" || exit 1 # location where the new files DB is built up and eventually zipped -TMPDIR="$(mktemp -d /tmp/create-filelists.tmpdir.XXXXXX)" || exit 1 +MYTMPDIR="$(mktemp -d ${WORKDIR}/create-filelists.tmpdir.XXXXXX)" || exit 1 #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null case "${DBEXT}" in - *.gz) TAR_OPT="z" ;; - *.bz2) TAR_OPT="j" ;; - *.xz) TAR_OPT="J" ;; - *) echo "Unknown compression type for DBEXT=${DBEXT}" && exit 1 ;; + *.gz) TAR_OPT="z" ;; + *.bz2) TAR_OPT="j" ;; + *.xz) TAR_OPT="J" ;; + *) echo "Unknown compression type for DBEXT=${DBEXT}" && exit 1 ;; esac FILESEXT="${DBEXT//db/files}" -for repo in $repos; do +for repo in ${repos[@]}; do REPO_DB_FILE="${repo}$DBEXT" FILES_DB_FILE="${repo}$FILESEXT" for arch in ${ARCHES[@]}; do -# echo "Running for architecture $arch, repo $repo" - cd "$reposdir" + pushd "$reposdir" >/dev/null repodir="${repo}/os/${arch}" cached="no" @@ -46,7 +40,6 @@ for repo in $repos; do # extract package db archive if [ -f "${targetdir}/${repodir}/${REPO_DB_FILE}" ]; then mkdir -p "${DBDIR}/${repodir}" -# echo "extracting $REPO_DB_FILE" bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${DBDIR}/${repodir}" else echo "Fail! Does the repo $repo with arch $arch even exist?" @@ -56,7 +49,6 @@ for repo in $repos; do # extract old file archive if [ -f "${targetdir}/${repodir}/${FILES_DB_FILE}" ]; then mkdir -p "${CACHEDIR}/${repodir}" -# echo "extracting $FILES_DB_FILE" bsdtar -xf "${targetdir}/${repodir}/${FILES_DB_FILE}" -C "${CACHEDIR}/${repodir}" cached="yes" fi @@ -65,15 +57,13 @@ for repo in $repos; do for pkg in $(ls ${DBDIR}/${repodir}); do dbpkgdir="${DBDIR}/${repodir}/${pkg}" cachepkgdir="${CACHEDIR}/${repodir}/${pkg}" - tmppkgdir="${TMPDIR}/${repodir}/${pkg}" + tmppkgdir="${MYTMPDIR}/${repodir}/${pkg}" mkdir -p "$tmppkgdir" ln "${dbpkgdir}/desc" "${tmppkgdir}/desc" ln "${dbpkgdir}/depends" "${tmppkgdir}/depends" if [ -f "${cachepkgdir}/files" ]; then -# echo "cache: $pkg" ln "${cachepkgdir}/files" "${tmppkgdir}/files" else -# echo "not cache: $repo/$arch: $pkg" filename=$(grep -A1 '^%FILENAME%$' "${dbpkgdir}/desc" | tail -n1) echo '%FILES%' > "${tmppkgdir}/files" bsdtar --exclude=.* -tf "$repodir/$filename" >> "${tmppkgdir}/files" @@ -84,19 +74,17 @@ for repo in $repos; do # create new file archive if [ "$cached" == "no" ]; then # at least one package has changed, so let's rebuild the archive -# echo "creating ${FILES_DB_FILE}/${arch}" pkgdir="${targetdir}/${repodir}" mkdir -p "$pkgdir" - cd "${TMPDIR}/${repodir}" + pushd "${MYTMPDIR}/${repodir}" >/dev/null [ -f "${pkgdir}/${FILES_DB_FILE}" ] && rm "${pkgdir}/${FILES_DB_FILE}" bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f "${pkgdir}/${FILES_DB_FILE}" * + popd >/dev/null fi + popd >/dev/null done done -cd - >/dev/null -rm -rf "$TMPDIR" "$CACHEDIR" "$DBDIR" -rm -f "$lock" || exit 1 -# echo 'done' +script_unlock # vim: set ts=4 sw=4 et ft=sh: diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 93cb842..78c3959 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -1,37 +1,16 @@ #!/bin/bash -ftpbase="/srv/ftp" -srcbase="/srv/ftp/sources" -repos="core extra testing community community-testing" - . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" -LOCKFILE="/tmp/.sourceball.lock" - -cleanup () { - rm -f "$LOCKFILE" - restore_umask - exit 0 -} - -ctrl_c() { - cleanup -} +ftpbase="${FTP_BASE}" +srcbase="${FTP_BASE}/sources" +repos=(core extra testing community community-testing) -if [ -f "$LOCKFILE" ]; then - owner="$(/usr/bin/stat -c %U $LOCKFILE)" - echo "error: source tarball generation is already in progress (started by $owner)" - exit 1 -fi - -trap cleanup 0 -trap ctrl_c 2 +script_lock set_umask -/bin/touch "$LOCKFILE" - dirname="$(/bin/readlink -f $(/usr/bin/dirname $0))" FAILED_PKGS="" @@ -39,7 +18,7 @@ FAILED_PKGS="" [ -e "$srcbase/errors.txt" ] && /bin/mv "$srcbase/errors.txt" "$srcbase/errors.txt.old" echo "Errors occured during run:" > "$srcbase/errors.txt" -for repo in $repos; do +for repo in ${repos[@]}; do for arch in ${ARCHES[@]} any; do ftppath="$ftpbase/$repo/os/$arch" if [ ! -d "$ftppath" ]; then @@ -91,4 +70,4 @@ fi $dirname/../misc-scripts/sourceballs-cleanup -cleanup +script_unlock -- cgit v1.2.3-2-g168b From af4f86808e8cd45cc171f55a1ec15bf30d858a0d Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 10 Aug 2010 21:40:24 +0200 Subject: Use more consitent naming for package pool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are no longer architecture-specific subdirs and the structure was switch to this: ftp └── pool ├── community └── packages packages contains all packages from core, extra and testing; this naming is in sync with the svn repo naming: svn-packages and svn-community --- cron-jobs/adjust-permissions | 8 ++++---- cron-jobs/ftpdir-cleanup | 20 +++++++++----------- 2 files changed, 13 insertions(+), 15 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index c21af0c..c28ca7c 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -17,9 +17,9 @@ get_dir_owner() { echo "root:tusers" ;; community-testing) echo "root:tusers" ;; - packages/arch) + pool/packages) echo "ftp:ftp-extra" ;; - packages/community) + pool/community) echo "root:tusers" ;; esac } @@ -35,8 +35,8 @@ for d in $(get_repos_for_host); do done for p in $(get_pkgpool_for_host); do owner="$(get_dir_owner $p)" - /bin/chown -R $owner $p/{any,i686,x86_64} - /bin/chmod -R g+w $p/{any,i686,x86_64} + /bin/chown $owner $p + /bin/chmod g+w $p done /bin/chmod 555 $FTP_BASE popd >/dev/null diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index f5e401a..7668ab5 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -17,18 +17,16 @@ for repo in $repos; do done to_cleanup="" -for _arch in any ${ARCHES[@]}; do - poolpath="$FTP_BASE/$(get_pkgpool_for_host)/$_arch/" - pushd $poolpath >/dev/null - for pkg in *$PKGEXT; do - [ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" - LINKS="$(/usr/bin/find $repopaths -type l -name "$pkg" 2>/dev/null)" - if [ -z "$LINKS" ]; then - to_cleanup="$to_cleanup $poolpath/$pkg" - fi - done - popd >/dev/null +poolpath="$FTP_BASE/$(get_pkgpool_for_host)/" +pushd $poolpath >/dev/null +for pkg in *$PKGEXT; do +[ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" +LINKS="$(/usr/bin/find $repopaths -type l -name "$pkg" 2>/dev/null)" +if [ -z "$LINKS" ]; then + to_cleanup="$to_cleanup $poolpath/$pkg" +fi done +popd >/dev/null if [ -n "$to_cleanup" ]; then echo " The following packages are no longer in any repo" -- cgit v1.2.3-2-g168b From a422060414670bb49d2422a38467b73ae01e7ecb Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Fri, 13 Aug 2010 09:47:31 +0200 Subject: Use common functions to print messages, warnings and errors These functions are copied from makepkg --- cron-jobs/create-filelists | 4 ++-- cron-jobs/ftpdir-cleanup | 8 +++----- cron-jobs/sourceballs | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index 8c354fc..9249408 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -23,7 +23,7 @@ case "${DBEXT}" in *.gz) TAR_OPT="z" ;; *.bz2) TAR_OPT="j" ;; *.xz) TAR_OPT="J" ;; - *) echo "Unknown compression type for DBEXT=${DBEXT}" && exit 1 ;; + *) die "Unknown compression type for DBEXT=${DBEXT}" ;; esac FILESEXT="${DBEXT//db/files}" @@ -42,7 +42,7 @@ for repo in ${repos[@]}; do mkdir -p "${DBDIR}/${repodir}" bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${DBDIR}/${repodir}" else - echo "Fail! Does the repo $repo with arch $arch even exist?" + error "Fail! Does the repo $repo with arch $arch even exist?" continue fi diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 7668ab5..3456e9c 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -29,13 +29,11 @@ done popd >/dev/null if [ -n "$to_cleanup" ]; then - echo " The following packages are no longer in any repo" - echo " They will be moved to $CLEANUP_DESTDIR" + msg "The following packages are no longer in any repo" for f in $to_cleanup; do - echo " $(basename "$f")" + msg2 "$(basename "$f")" done - echo "" - mv $to_cleanup "$CLEANUP_DESTDIR" + ${CLEANUP_DRYRUN} || mv $to_cleanup "$CLEANUP_DESTDIR" fi script_unlock diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 78c3959..b82e375 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -22,7 +22,7 @@ for repo in ${repos[@]}; do for arch in ${ARCHES[@]} any; do ftppath="$ftpbase/$repo/os/$arch" if [ ! -d "$ftppath" ]; then - echo "FTP path does not exist: $ftppath" >2 + error "FTP path does not exist: $ftppath" continue fi cd $ftppath -- cgit v1.2.3-2-g168b From 4daeb0a71e65618c56d672118675c9efd57c5979 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 17 Aug 2010 08:00:21 +0200 Subject: Add [staging] repository --- cron-jobs/adjust-permissions | 2 ++ cron-jobs/create-filelists | 2 +- cron-jobs/sourceballs | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index c28ca7c..6acfcf2 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -13,6 +13,8 @@ get_dir_owner() { echo "ftp:ftp-extra" ;; testing) echo "ftp:ftp-extra" ;; + staging) + echo "ftp:ftp-extra" ;; community) echo "root:tusers" ;; community-testing) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index 9249408..fd07617 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -5,7 +5,7 @@ reposdir=${FTP_BASE} targetdir=${FTP_BASE} -repos=(core extra testing community community-testing) +repos=(core extra testing community community-testing staging) script_lock diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index b82e375..a6f7a19 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -5,7 +5,7 @@ ftpbase="${FTP_BASE}" srcbase="${FTP_BASE}/sources" -repos=(core extra testing community community-testing) +repos=(core extra testing community community-testing staging) script_lock -- cgit v1.2.3-2-g168b From 683db123710ff9092ae99929fce232620932fdb0 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Wed, 18 Aug 2010 16:20:27 +0200 Subject: Rewrite ftpdir-cleanup * runtime reduced to < 1 second * uses file lists and the comm command * searches for missing packages * removes empty legacy directories like extra/os/any --- cron-jobs/ftpdir-cleanup | 116 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 92 insertions(+), 24 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 3456e9c..0a43bd1 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -3,37 +3,105 @@ . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" -repos="$(get_repos_for_host)" +clean_pkg() { + local pkg + local target + + if ! ${CLEANUP_DRYRUN}; then + for pkg in "$@"; do + if [ -h "$pkg" ]; then + rm -f "$pkg" + else + mv -f "$pkg" "$CLEANUP_DESTDIR" + fi + done + fi +} + +repos=($(get_repos_for_host)) script_lock -#adjust the nice level to run at a lower priority -/usr/bin/renice +10 -p $$ > /dev/null +for repo in ${repos[@]}; do + for arch in ${ARCHES[@]}; do + repo_lock ${repo} ${arch} || exit 1 + done +done + +${CLEANUP_DRYRUN} && warning 'dry run mode is active' -repopaths='' -for repo in $repos; do - $(dirname $0)/../misc-scripts/ftpdir-cleanup-repo $repo - repopaths="${repopaths} ${FTP_BASE}/${repo}/os/" +for repo in ${repos[@]}; do + for arch in ${ARCHES[@]}; do + if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + warning "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} not found, skipping" + continue + fi + # get a list of actual available package files + find "${FTP_BASE}/${repo}/os/${arch}" -xtype f -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/repo-${repo}-${arch}" + # get a list of package files defined in the repo db + bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/db-${repo}-${arch}" + + missing_pkgs=($(comm -13 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) + if [ ${#missing_pkgs[@]} -ge 1 ]; then + error "Missing packages in [${repo}] (${arch})..." + for missing_pkg in ${missing_pkgs[@]}; do + msg2 "${missing_pkg}" + done + fi + + old_pkgs=($(comm -23 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) + if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old packages from [${repo}] (${arch})..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + clean_pkg "${FTP_BASE}/${repo}/os/${arch}/${old_pkg}" + done + fi + done done -to_cleanup="" -poolpath="$FTP_BASE/$(get_pkgpool_for_host)/" -pushd $poolpath >/dev/null -for pkg in *$PKGEXT; do -[ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" -LINKS="$(/usr/bin/find $repopaths -type l -name "$pkg" 2>/dev/null)" -if [ -z "$LINKS" ]; then - to_cleanup="$to_cleanup $poolpath/$pkg" +# get a list of all available packages in the pacakge pool +find "$FTP_BASE/$(get_pkgpool_for_host)" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/pool" +# create a list of packages in our db +cat "${WORKDIR}/db-"* | sort -u > "${WORKDIR}/db" + +old_pkgs=($(comm -23 "${WORKDIR}/pool" "${WORKDIR}/db")) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old packages from package pool..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + clean_pkg "$FTP_BASE/$(get_pkgpool_for_host)/${old_pkg}" + done fi + +# cleanup of legacy $repo/os/any directories +for repo in ${repos[@]}; do + if [ ! -d "${FTP_BASE}/${repo}/os/any" ]; then + continue + fi + if [ -n "$(find "${FTP_BASE}/${repo}/os/any" -type d -empty)" ]; then + msg "Removing empty legacy directory ${repo}/os/any" + ${CLEANUP_DRYRUN} && rmdir "${FTP_BASE}/${repo}/os/any" + continue + fi + find "${FTP_BASE}/${repo}/os/any" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/any-${repo}" + cat "${WORKDIR}/db-${repo}-"* | sort -u > "${WORKDIR}/all-${repo}" + + old_pkgs=($(comm -23 "${WORKDIR}/any-${repo}" "${WORKDIR}/all-${repo}")) + if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old packages from [${repo}] (any)..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + clean_pkg "${FTP_BASE}/${repo}/os/any/${old_pkg}" + done + fi +done + + +for repo in ${repos[@]}; do + for arch in ${ARCHES[@]}; do + repo_unlock ${repo} ${arch} + done done -popd >/dev/null - -if [ -n "$to_cleanup" ]; then - msg "The following packages are no longer in any repo" - for f in $to_cleanup; do - msg2 "$(basename "$f")" - done - ${CLEANUP_DRYRUN} || mv $to_cleanup "$CLEANUP_DESTDIR" -fi script_unlock -- cgit v1.2.3-2-g168b From b48b6b62b70fda2cc0318668eeda457c0f8d2123 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Wed, 18 Aug 2010 16:38:21 +0200 Subject: fix typo --- cron-jobs/ftpdir-cleanup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 0a43bd1..5c8776e 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -81,7 +81,7 @@ for repo in ${repos[@]}; do fi if [ -n "$(find "${FTP_BASE}/${repo}/os/any" -type d -empty)" ]; then msg "Removing empty legacy directory ${repo}/os/any" - ${CLEANUP_DRYRUN} && rmdir "${FTP_BASE}/${repo}/os/any" + ${CLEANUP_DRYRUN} || rmdir "${FTP_BASE}/${repo}/os/any" continue fi find "${FTP_BASE}/${repo}/os/any" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/any-${repo}" -- cgit v1.2.3-2-g168b From 9e8897aeadb2aa51b70c5fc0ed1ae281b015d146 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 22 Aug 2010 22:01:09 +0200 Subject: add makepkg.conf for [multilib] --- cron-jobs/adjust-permissions | 2 ++ cron-jobs/create-filelists | 2 +- cron-jobs/sourceballs | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index 6acfcf2..2cc8b40 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -19,6 +19,8 @@ get_dir_owner() { echo "root:tusers" ;; community-testing) echo "root:tusers" ;; + multilib) + echo "root:tusers" ;; pool/packages) echo "ftp:ftp-extra" ;; pool/community) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index fd07617..e280f84 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -5,7 +5,7 @@ reposdir=${FTP_BASE} targetdir=${FTP_BASE} -repos=(core extra testing community community-testing staging) +repos=(core extra testing community community-testing staging multilib) script_lock diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index a6f7a19..a482eb7 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -5,7 +5,7 @@ ftpbase="${FTP_BASE}" srcbase="${FTP_BASE}/sources" -repos=(core extra testing community community-testing staging) +repos=(core extra testing community community-testing staging multilib) script_lock -- cgit v1.2.3-2-g168b From 9f695214d0d63f60d67bb5c1caeae50110c03f7a Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 24 Aug 2010 20:23:30 +0200 Subject: adjust group for multilib repository --- cron-jobs/adjust-permissions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index 2cc8b40..a68584c 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -20,7 +20,7 @@ get_dir_owner() { community-testing) echo "root:tusers" ;; multilib) - echo "root:tusers" ;; + echo "root:multilib" ;; pool/packages) echo "ftp:ftp-extra" ;; pool/community) -- cgit v1.2.3-2-g168b From 0e9dc9f69e290fe352de7b2bbd288890a78eced5 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 25 Aug 2010 11:30:42 -0500 Subject: Add update-web-db cronjob This has been untracked by version control for a while on gerolde in both the cron-jobs directory and in /etc/cron.hourly/. Add it here so we can make changes and know what is going on. This is an improved script over what we currently have. It is one script instead of two, and it does things a little smarter with the logging. Signed-off-by: Dan McGee --- cron-jobs/update-web-db | 47 +++++++++++++++++++++++++++++++++++++++++++ cron-jobs/update-web-files-db | 1 + 2 files changed, 48 insertions(+) create mode 100755 cron-jobs/update-web-db create mode 120000 cron-jobs/update-web-files-db (limited to 'cron-jobs') diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db new file mode 100755 index 0000000..8631a04 --- /dev/null +++ b/cron-jobs/update-web-db @@ -0,0 +1,47 @@ +#!/bin/bash + +# run at nice 5. it can churn quite a bit of cpu after all. +renice +5 -p $$ > /dev/null + +# setup paths +SPATH="/srv/http/archweb" +ARCHES="i686 x86_64" +# having "more important repos" last should make [core] trickle to the top of +# the updates list each hour rather than being overwhelmed by big [extra] and +# [community] updates +REPOS="community-testing multilib community extra testing core" +LOGOUT="/tmp/archweb_update.log" + +# figure out what operation to perform +cmd="$(basename $0)" +if [[ $cmd != "update-web-db" && $cmd != "update-web-files-db" ]]; then + echo "Invalid command name '$cmd' specified!" + exit 1 +fi + +echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" + +## do the dirty, then output the result to a file. +cd $SPATH +for arch in ${ARCHES}; do + for repo in ${REPOS}; do + echo "Updating ${repo}-${arch}" + case "$cmd" in + update-web-db) + dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.db.tar.gz" + flags="" ;; + update-web-files-db) + dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.files.tar.gz" + flags="--filesonly" ;; + esac + ./manage.py reporead ${flags} ${arch} ${dbfile} + echo "" + done +done >> "${LOGOUT}" 2>&1 + +echo "" >> "${LOGOUT}" + +# rotate the file if it is getting big (> 10M), overwriting any old backup +if [[ $(stat -c%s update-web-db) -gt 10485760 ]]; then + mv "${LOGOUT}" "${LOGOUT}.old" +fi diff --git a/cron-jobs/update-web-files-db b/cron-jobs/update-web-files-db new file mode 120000 index 0000000..0c2c4fa --- /dev/null +++ b/cron-jobs/update-web-files-db @@ -0,0 +1 @@ +update-web-db \ No newline at end of file -- cgit v1.2.3-2-g168b From 960917cdb0b23163029dc81e52f295e3281492e1 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 25 Aug 2010 11:38:10 -0500 Subject: Fix a few issues with update-web-db Signed-off-by: Dan McGee --- cron-jobs/update-web-db | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 8631a04..4d1519c 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -25,7 +25,6 @@ echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" cd $SPATH for arch in ${ARCHES}; do for repo in ${REPOS}; do - echo "Updating ${repo}-${arch}" case "$cmd" in update-web-db) dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.db.tar.gz" @@ -34,14 +33,17 @@ for arch in ${ARCHES}; do dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.files.tar.gz" flags="--filesonly" ;; esac - ./manage.py reporead ${flags} ${arch} ${dbfile} - echo "" + if [[ -f $dbfile ]]; then + echo "Updating ${repo}-${arch}" + ./manage.py reporead ${flags} ${arch} ${dbfile} + echo "" + fi done -done >> "${LOGOUT}" 2>&1 +done >> "${LOGOUT}" 2>&1 echo "" >> "${LOGOUT}" # rotate the file if it is getting big (> 10M), overwriting any old backup -if [[ $(stat -c%s update-web-db) -gt 10485760 ]]; then +if [[ $(stat -c%s "${LOGOUT}") -gt 10485760 ]]; then mv "${LOGOUT}" "${LOGOUT}.old" fi -- cgit v1.2.3-2-g168b From 1d6eb15bcc4c355edef1d6c249e9c67bca7d0404 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 25 Aug 2010 11:39:16 -0500 Subject: update-web-db: reformat to coding style Signed-off-by: Dan McGee --- cron-jobs/update-web-db | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 4d1519c..3504813 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -15,8 +15,8 @@ LOGOUT="/tmp/archweb_update.log" # figure out what operation to perform cmd="$(basename $0)" if [[ $cmd != "update-web-db" && $cmd != "update-web-files-db" ]]; then - echo "Invalid command name '$cmd' specified!" - exit 1 + echo "Invalid command name '$cmd' specified!" + exit 1 fi echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" @@ -25,19 +25,19 @@ echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" cd $SPATH for arch in ${ARCHES}; do for repo in ${REPOS}; do - case "$cmd" in - update-web-db) - dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.db.tar.gz" - flags="" ;; - update-web-files-db) - dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.files.tar.gz" - flags="--filesonly" ;; - esac - if [[ -f $dbfile ]]; then - echo "Updating ${repo}-${arch}" - ./manage.py reporead ${flags} ${arch} ${dbfile} - echo "" - fi + case "$cmd" in + update-web-db) + dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.db.tar.gz" + flags="" ;; + update-web-files-db) + dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.files.tar.gz" + flags="--filesonly" ;; + esac + if [[ -f $dbfile ]]; then + echo "Updating ${repo}-${arch}" + ./manage.py reporead ${flags} ${arch} ${dbfile} + echo "" + fi done done >> "${LOGOUT}" 2>&1 @@ -45,5 +45,7 @@ echo "" >> "${LOGOUT}" # rotate the file if it is getting big (> 10M), overwriting any old backup if [[ $(stat -c%s "${LOGOUT}") -gt 10485760 ]]; then - mv "${LOGOUT}" "${LOGOUT}.old" + mv "${LOGOUT}" "${LOGOUT}.old" fi + +# vim: set ts=4 sw=4 et ft=sh: -- cgit v1.2.3-2-g168b From 3c07e2e3ff951242a5d9fc124c6937e75e961da2 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 25 Aug 2010 11:39:57 -0500 Subject: update-web-db: flip the arch/repo loops This will really make sure [core] packages flow to the top of the list, regardless of architecture. They should anyway, but anything that was only updated in the first architecture updated could get artificially pushed down before. Signed-off-by: Dan McGee --- cron-jobs/update-web-db | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 3504813..8f91ca2 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -23,8 +23,8 @@ echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" ## do the dirty, then output the result to a file. cd $SPATH -for arch in ${ARCHES}; do - for repo in ${REPOS}; do +for repo in ${REPOS}; do + for arch in ${ARCHES}; do case "$cmd" in update-web-db) dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.db.tar.gz" -- cgit v1.2.3-2-g168b From 328b1ce478e25902aba5d8f19024dadeaaf3f678 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Wed, 1 Sep 2010 19:34:47 +0200 Subject: Simplify repo configuration * Repositories can now be defined in the config file for each host * added community-staging, gnome-unstable and kde-unstable * Exception is the adjust-permission cron-job; but we might want to use acls in future anyway Signed-off-by: Pierre Schmitz --- cron-jobs/adjust-permissions | 6 ++++++ cron-jobs/create-filelists | 3 +-- cron-jobs/integrity-check | 2 +- cron-jobs/sourceballs | 3 +-- 4 files changed, 9 insertions(+), 5 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index a68584c..e0b6e4e 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -15,10 +15,16 @@ get_dir_owner() { echo "ftp:ftp-extra" ;; staging) echo "ftp:ftp-extra" ;; + kde-unstable) + echo "ftp:ftp-extra" ;; + gnome-unstable) + echo "ftp:ftp-extra" ;; community) echo "root:tusers" ;; community-testing) echo "root:tusers" ;; + community-staging) + echo "root:tusers" ;; multilib) echo "root:multilib" ;; pool/packages) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index e280f84..a37080f 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -5,7 +5,6 @@ reposdir=${FTP_BASE} targetdir=${FTP_BASE} -repos=(core extra testing community community-testing staging multilib) script_lock @@ -28,7 +27,7 @@ esac FILESEXT="${DBEXT//db/files}" -for repo in ${repos[@]}; do +for repo in ${PKGREPO[@]}; do REPO_DB_FILE="${repo}$DBEXT" FILES_DB_FILE="${repo}$FILESEXT" for arch in ${ARCHES[@]}; do diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index 0b2b9c7..c120cfd 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -7,7 +7,7 @@ if [ $# -ne 2 ]; then exit 1 fi -for arch in i686 x86_64; do +for arch in ${ARCHES[@]}; do $basedir/check_archlinux/check_packages.py \ --repos="$1" \ --abs-tree="/srv/abs/rsync/$arch,/srv/abs/rsync/any" --arch="$arch" |\ diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index a482eb7..0cf062f 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -5,7 +5,6 @@ ftpbase="${FTP_BASE}" srcbase="${FTP_BASE}/sources" -repos=(core extra testing community community-testing staging multilib) script_lock @@ -18,7 +17,7 @@ FAILED_PKGS="" [ -e "$srcbase/errors.txt" ] && /bin/mv "$srcbase/errors.txt" "$srcbase/errors.txt.old" echo "Errors occured during run:" > "$srcbase/errors.txt" -for repo in ${repos[@]}; do +for repo in ${PKGREPO[@]}; do for arch in ${ARCHES[@]} any; do ftppath="$ftpbase/$repo/os/$arch" if [ ! -d "$ftppath" ]; then -- cgit v1.2.3-2-g168b From df78f6e877325ddfd9569e9704612ee2509791b1 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 2 Sep 2010 00:45:38 -0500 Subject: Simplify case statement Don't duplicate the user:group pair all over the place; put all of the associated repos and directories in the same case statement for a given set of credentials. Signed-off-by: Dan McGee Signed-off-by: Pierre Schmitz --- cron-jobs/adjust-permissions | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index e0b6e4e..6510d79 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -7,30 +7,17 @@ script_lock get_dir_owner() { case $1 in + # on main server, e.g. gerolde core) echo "ftp:ftp-arch" ;; - extra) + extra|testing|staging|kde-unstable|gnome-unstable|pool/packages) echo "ftp:ftp-extra" ;; - testing) - echo "ftp:ftp-extra" ;; - staging) - echo "ftp:ftp-extra" ;; - kde-unstable) - echo "ftp:ftp-extra" ;; - gnome-unstable) - echo "ftp:ftp-extra" ;; - community) - echo "root:tusers" ;; - community-testing) - echo "root:tusers" ;; - community-staging) + + # on community server, e.g. sigurd + community|community-testing|community-staging|pool/community) echo "root:tusers" ;; multilib) echo "root:multilib" ;; - pool/packages) - echo "ftp:ftp-extra" ;; - pool/community) - echo "root:tusers" ;; esac } -- cgit v1.2.3-2-g168b From e24a59e12360c4c807dde8c24f26035b4cd24287 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 11 Sep 2010 18:33:21 +0200 Subject: Remove obsolete cron job --- cron-jobs/adjust-permissions | 41 ----------------------------------------- 1 file changed, 41 deletions(-) delete mode 100755 cron-jobs/adjust-permissions (limited to 'cron-jobs') diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions deleted file mode 100755 index 6510d79..0000000 --- a/cron-jobs/adjust-permissions +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -script_lock - -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" - -get_dir_owner() { - case $1 in - # on main server, e.g. gerolde - core) - echo "ftp:ftp-arch" ;; - extra|testing|staging|kde-unstable|gnome-unstable|pool/packages) - echo "ftp:ftp-extra" ;; - - # on community server, e.g. sigurd - community|community-testing|community-staging|pool/community) - echo "root:tusers" ;; - multilib) - echo "root:multilib" ;; - esac -} - -#adjust the nice level to run at a lower priority -/usr/bin/renice +10 -p $$ > /dev/null - -pushd $FTP_BASE >/dev/null -for d in $(get_repos_for_host); do - owner="$(get_dir_owner $d)" - /bin/chown -R $owner $d/os/{any,i686,x86_64} - /bin/chmod -R g+w $d/os/{any,i686,x86_64} -done -for p in $(get_pkgpool_for_host); do - owner="$(get_dir_owner $p)" - /bin/chown $owner $p - /bin/chmod g+w $p -done -/bin/chmod 555 $FTP_BASE -popd >/dev/null - -script_unlock -- cgit v1.2.3-2-g168b From f121126f8166fb6dc261ea82f2890ba6693d047e Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 11 Sep 2010 22:52:23 +0200 Subject: Use local config instead of guessing by hostname Using the hostname to decide which repos to use is not releiable and hard to test. Instead use config.local to configure these. config files for sigurd and gerolde were added which can be copied or symlinked to config.local on the specific host. --- cron-jobs/ftpdir-cleanup | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 5c8776e..914ef8c 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -18,11 +18,9 @@ clean_pkg() { fi } -repos=($(get_repos_for_host)) - script_lock -for repo in ${repos[@]}; do +for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do repo_lock ${repo} ${arch} || exit 1 done @@ -30,7 +28,7 @@ done ${CLEANUP_DRYRUN} && warning 'dry run mode is active' -for repo in ${repos[@]}; do +for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then warning "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} not found, skipping" @@ -61,7 +59,7 @@ for repo in ${repos[@]}; do done # get a list of all available packages in the pacakge pool -find "$FTP_BASE/$(get_pkgpool_for_host)" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/pool" +find "$FTP_BASE/${PKGPOOL}" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/pool" # create a list of packages in our db cat "${WORKDIR}/db-"* | sort -u > "${WORKDIR}/db" @@ -70,12 +68,12 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old packages from package pool..." for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" - clean_pkg "$FTP_BASE/$(get_pkgpool_for_host)/${old_pkg}" + clean_pkg "$FTP_BASE/${PKGPOOL}/${old_pkg}" done fi # cleanup of legacy $repo/os/any directories -for repo in ${repos[@]}; do +for repo in ${PKGREPOS[@]}; do if [ ! -d "${FTP_BASE}/${repo}/os/any" ]; then continue fi @@ -98,7 +96,7 @@ for repo in ${repos[@]}; do done -for repo in ${repos[@]}; do +for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do repo_unlock ${repo} ${arch} done -- cgit v1.2.3-2-g168b From 8a5f6e5542590edb94499bd6f36ba3787efbc2ce Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 12 Sep 2010 09:50:44 -0500 Subject: Add multilib-testing to web update script Signed-off-by: Dan McGee --- cron-jobs/update-web-db | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 8f91ca2..8b74a3a 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -9,7 +9,7 @@ ARCHES="i686 x86_64" # having "more important repos" last should make [core] trickle to the top of # the updates list each hour rather than being overwhelmed by big [extra] and # [community] updates -REPOS="community-testing multilib community extra testing core" +REPOS="community-testing multilib-testing multilib community extra testing core" LOGOUT="/tmp/archweb_update.log" # figure out what operation to perform -- cgit v1.2.3-2-g168b From d9efa9b7bbbc854b987f158cd19d5f646594e42b Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Mon, 13 Sep 2010 23:08:20 +0200 Subject: Fix typos --- cron-jobs/create-filelists | 2 +- cron-jobs/sourceballs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index a37080f..2526a61 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -27,7 +27,7 @@ esac FILESEXT="${DBEXT//db/files}" -for repo in ${PKGREPO[@]}; do +for repo in ${PKGREPOS[@]}; do REPO_DB_FILE="${repo}$DBEXT" FILES_DB_FILE="${repo}$FILESEXT" for arch in ${ARCHES[@]}; do diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 0cf062f..0b01321 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -17,7 +17,7 @@ FAILED_PKGS="" [ -e "$srcbase/errors.txt" ] && /bin/mv "$srcbase/errors.txt" "$srcbase/errors.txt.old" echo "Errors occured during run:" > "$srcbase/errors.txt" -for repo in ${PKGREPO[@]}; do +for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]} any; do ftppath="$ftpbase/$repo/os/$arch" if [ ! -d "$ftppath" ]; then -- cgit v1.2.3-2-g168b From 44c24c5d8e98eb51c8612dfaaac3ddd2ce0337dd Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 16 Sep 2010 18:50:53 -0500 Subject: Move output redirection Signed-off-by: Dan McGee --- cron-jobs/update-web-db | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 8b74a3a..275676e 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -34,12 +34,12 @@ for repo in ${REPOS}; do flags="--filesonly" ;; esac if [[ -f $dbfile ]]; then - echo "Updating ${repo}-${arch}" - ./manage.py reporead ${flags} ${arch} ${dbfile} + echo "Updating ${repo}-${arch}" >> "${LOGOUT}" + ./manage.py reporead ${flags} ${arch} ${dbfile} >> "${LOGOUT}" 2>&1 echo "" fi done -done >> "${LOGOUT}" 2>&1 +done echo "" >> "${LOGOUT}" -- cgit v1.2.3-2-g168b From 7711a02ed5ce1be7f5f0cb572a1efc4c29b6dfe0 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 11 Oct 2010 12:28:52 -0500 Subject: Fix missing output redirection Signed-off-by: Dan McGee --- cron-jobs/update-web-db | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 275676e..3f4a2fc 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -36,7 +36,7 @@ for repo in ${REPOS}; do if [[ -f $dbfile ]]; then echo "Updating ${repo}-${arch}" >> "${LOGOUT}" ./manage.py reporead ${flags} ${arch} ${dbfile} >> "${LOGOUT}" 2>&1 - echo "" + echo "" >> "${LOGOUT}" fi done done -- cgit v1.2.3-2-g168b From a1ba979a0e3c763449d91fc787333f4058e89aa8 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 12 Oct 2010 11:17:07 +0200 Subject: Dont try to index empty repos --- cron-jobs/create-filelists | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index 2526a61..d8a5ba1 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -41,7 +41,7 @@ for repo in ${PKGREPOS[@]}; do mkdir -p "${DBDIR}/${repodir}" bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${DBDIR}/${repodir}" else - error "Fail! Does the repo $repo with arch $arch even exist?" + warning "The repo $repo with arch $arch does not exist" continue fi @@ -71,7 +71,7 @@ for repo in ${PKGREPOS[@]}; do done # create new file archive - if [ "$cached" == "no" ]; then + if [ "$cached" == "no" -a -d "${MYTMPDIR}/${repodir}" ]; then # at least one package has changed, so let's rebuild the archive pkgdir="${targetdir}/${repodir}" mkdir -p "$pkgdir" -- cgit v1.2.3-2-g168b From cb2dcc6ee207e9c5ba4b875d70e387e6347591ed Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 20 Nov 2010 16:11:59 +0100 Subject: Fix sourceballs cron job * add unit test for sourceballs and cleanup * introduce SRCPOOL and LOGDIR variables in config --- cron-jobs/sourceballs | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 0b01321..4fc194f 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -3,9 +3,6 @@ . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" -ftpbase="${FTP_BASE}" -srcbase="${FTP_BASE}/sources" - script_lock set_umask @@ -14,12 +11,12 @@ dirname="$(/bin/readlink -f $(/usr/bin/dirname $0))" FAILED_PKGS="" -[ -e "$srcbase/errors.txt" ] && /bin/mv "$srcbase/errors.txt" "$srcbase/errors.txt.old" -echo "Errors occured during run:" > "$srcbase/errors.txt" +[ ! -d "${LOGDIR}/sourceballs" ] && mkdir -p "${LOGDIR}/sourceballs" +[ -e "${LOGDIR}/sourceballs/errors.txt" ] && /bin/mv "${LOGDIR}/sourceballs/errors.txt" "${LOGDIR}/sourceballs/errors.txt.old" for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]} any; do - ftppath="$ftpbase/$repo/os/$arch" + for arch in ${ARCHES[@]}; do + ftppath="${FTP_BASE}/$repo/os/$arch" if [ ! -d "$ftppath" ]; then error "FTP path does not exist: $ftppath" continue @@ -28,7 +25,6 @@ for repo in ${PKGREPOS[@]}; do for pkg in *$PKGEXT; do [ -f "$pkg" ] || continue pkgbase=$(getpkgbase $pkg) - srcpath="$srcbase/" srcpkg="${pkg//$PKGEXT/$SRCEXT}" srcpkg="${srcpkg//-$arch/}" srcpkgname="${srcpkg%-*-*$SRCEXT}" @@ -51,9 +47,9 @@ for repo in ${PKGREPOS[@]}; do force="-f" fi - if [ ! \( -f "$srcpath$srcpkg" -o -f "$srcpath$srcpkgbase" \) ]; then + if [ ! \( -f "${FTP_BASE}/${SRCPOOL}/$srcpkg" -o -f "${FTP_BASE}/${SRCPOOL}/$srcpkgbase" \) ]; then if ! $dirname/../misc-scripts/make-sourceball $force \ - $pkgbase $repo $arch 2>>"$srcbase/errors.txt"; then + $pkgbase $repo $arch 2>>"${LOGDIR}/sourceballs/errors.txt"; then FAILED_PKGS="$FAILED_PKGS $pkgbase" fi fi @@ -62,9 +58,8 @@ for repo in ${PKGREPOS[@]}; do done if [ -n "$FAILED_PKGS" ]; then - [ -e "$srcbase/failed.txt" ] && /bin/mv "$srcbase/failed.txt" "$srcbase/failed.txt.old" - echo "The following package bases failed:" > "$srcbase/failed.txt" - echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u >> "$srcbase/failed.txt" + [ -e "${LOGDIR}/sourceballs/failed.txt" ] && /bin/mv "${LOGDIR}/sourceballs/failed.txt" "${LOGDIR}/sourceballs/failed.txt.old" + echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u >> "${LOGDIR}/sourceballs/failed.txt" fi $dirname/../misc-scripts/sourceballs-cleanup -- cgit v1.2.3-2-g168b From a7591f4be3f9e741f5d1e5aeadd3ab20b497a252 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 20 Nov 2010 20:38:38 +0100 Subject: Simplify sourceballs creation * Read package lists directly from DB file * Make SVNREPO configurable --- cron-jobs/sourceballs | 46 +++++++++++++++++----------------------------- 1 file changed, 17 insertions(+), 29 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 4fc194f..1add36e 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -9,59 +9,47 @@ set_umask dirname="$(/bin/readlink -f $(/usr/bin/dirname $0))" -FAILED_PKGS="" - [ ! -d "${LOGDIR}/sourceballs" ] && mkdir -p "${LOGDIR}/sourceballs" [ -e "${LOGDIR}/sourceballs/errors.txt" ] && /bin/mv "${LOGDIR}/sourceballs/errors.txt" "${LOGDIR}/sourceballs/errors.txt.old" +[ -e "${LOGDIR}/sourceballs/failed.txt" ] && /bin/mv "${LOGDIR}/sourceballs/failed.txt" "${LOGDIR}/sourceballs/failed.txt.old" for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do ftppath="${FTP_BASE}/$repo/os/$arch" - if [ ! -d "$ftppath" ]; then - error "FTP path does not exist: $ftppath" + dbfile="${ftppath}/${repo}${DBEXT}" + if [ ! -r "${dbfile}" ]; then + warning "DB file does not exist: ${dbfile}" continue fi - cd $ftppath - for pkg in *$PKGEXT; do - [ -f "$pkg" ] || continue - pkgbase=$(getpkgbase $pkg) - srcpkg="${pkg//$PKGEXT/$SRCEXT}" - srcpkg="${srcpkg//-$arch/}" - srcpkgname="${srcpkg%-*-*$SRCEXT}" - srcpkgbase="${srcpkg/$srcpkgname/$pkgbase}" - - #Don't do anything for package in this 'blacklist' - if grep "^$pkgbase\$" "$dirname/sourceballs.skip" >/dev/null 2>&1; then - continue - fi - - #This pkgbase has already failed. No sense in trying it again this run - if echo $FAILED_PKGS | grep "\<$pkgbase\>" >&/dev/null; then + for pkg in $(bsdtar -xOf "${dbfile}" \ + | awk '/^%NAME%/{getline b};/^%BASE%/{getline b};/^%VERSION%/{getline v};/^%ARCH%/{printf "%s/%s\n", b, v}' \ + | sort -u); do + pkgbase=${pkg%/*} + pkgver=${pkg#*/} + srcpkg="${pkgbase}-${pkgver}${SRCEXT}" + + # Don't do anything for package in this 'blacklist' + if grep -q "^${pkgbase}\$" "$dirname/sourceballs.skip"; then continue fi - #Use this file to 'whitelist' or force building some sourceballs, + # Use this file to 'whitelist' or force building some sourceballs, # skipping the license check force="" - if grep "^$pkgbase\$" "$dirname/sourceballs.force" >/dev/null 2>&1; then + if grep -q "^$pkgbase\$" "$dirname/sourceballs.force"; then force="-f" fi - if [ ! \( -f "${FTP_BASE}/${SRCPOOL}/$srcpkg" -o -f "${FTP_BASE}/${SRCPOOL}/$srcpkgbase" \) ]; then + if [ ! -f "${FTP_BASE}/${SRCPOOL}/$srcpkg" ]; then if ! $dirname/../misc-scripts/make-sourceball $force \ $pkgbase $repo $arch 2>>"${LOGDIR}/sourceballs/errors.txt"; then - FAILED_PKGS="$FAILED_PKGS $pkgbase" + echo "$pkgbase" >> "${LOGDIR}/sourceballs/failed.txt" fi fi done done done -if [ -n "$FAILED_PKGS" ]; then - [ -e "${LOGDIR}/sourceballs/failed.txt" ] && /bin/mv "${LOGDIR}/sourceballs/failed.txt" "${LOGDIR}/sourceballs/failed.txt.old" - echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u >> "${LOGDIR}/sourceballs/failed.txt" -fi - $dirname/../misc-scripts/sourceballs-cleanup script_unlock -- cgit v1.2.3-2-g168b From a2cdd819ed32cd4ccfefab2c2ce0ead118f9772d Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 20 Nov 2010 21:29:47 +0100 Subject: sourceballs: Lock the repo while reading the db file --- cron-jobs/sourceballs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 1add36e..351b06d 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -21,9 +21,16 @@ for repo in ${PKGREPOS[@]}; do warning "DB file does not exist: ${dbfile}" continue fi - for pkg in $(bsdtar -xOf "${dbfile}" \ - | awk '/^%NAME%/{getline b};/^%BASE%/{getline b};/^%VERSION%/{getline v};/^%ARCH%/{printf "%s/%s\n", b, v}' \ - | sort -u); do + + repo_lock ${repo} ${arch} || exit 1 + # Read packages from db file + # Format is: /- + pkgs=($(bsdtar -xOf "${dbfile}" \ + | awk '/^%NAME%/{getline b};/^%BASE%/{getline b};/^%VERSION%/{getline v};/^%ARCH%/{printf "%s/%s ", b, v}' \ + | sort -u)) + repo_unlock ${repo} ${arch} + + for pkg in ${pkgs[@]}; do pkgbase=${pkg%/*} pkgver=${pkg#*/} srcpkg="${pkgbase}-${pkgver}${SRCEXT}" -- cgit v1.2.3-2-g168b From 30a128a864bdbfc294b6ba6a49c9264570bb3c58 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Nov 2010 13:23:09 +0100 Subject: sourceballs: Don't write any log This is consistent with the other dbscripts. The output will be send to the mailinglist. --- cron-jobs/sourceballs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 351b06d..23e8421 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -9,16 +9,12 @@ set_umask dirname="$(/bin/readlink -f $(/usr/bin/dirname $0))" -[ ! -d "${LOGDIR}/sourceballs" ] && mkdir -p "${LOGDIR}/sourceballs" -[ -e "${LOGDIR}/sourceballs/errors.txt" ] && /bin/mv "${LOGDIR}/sourceballs/errors.txt" "${LOGDIR}/sourceballs/errors.txt.old" -[ -e "${LOGDIR}/sourceballs/failed.txt" ] && /bin/mv "${LOGDIR}/sourceballs/failed.txt" "${LOGDIR}/sourceballs/failed.txt.old" - for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do ftppath="${FTP_BASE}/$repo/os/$arch" dbfile="${ftppath}/${repo}${DBEXT}" if [ ! -r "${dbfile}" ]; then - warning "DB file does not exist: ${dbfile}" + warning "${dbfile} not found, skipping" continue fi @@ -48,9 +44,8 @@ for repo in ${PKGREPOS[@]}; do fi if [ ! -f "${FTP_BASE}/${SRCPOOL}/$srcpkg" ]; then - if ! $dirname/../misc-scripts/make-sourceball $force \ - $pkgbase $repo $arch 2>>"${LOGDIR}/sourceballs/errors.txt"; then - echo "$pkgbase" >> "${LOGDIR}/sourceballs/failed.txt" + if ! $dirname/../misc-scripts/make-sourceball $force $pkgbase $repo $arch; then + error "Failed to download sources for $pkgbase" fi fi done -- cgit v1.2.3-2-g168b From 952f8dc9ddbce506e4943afd8e053feebfdf9313 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Nov 2010 14:00:27 +0100 Subject: sourceballs.skip: nexuiz-data has the same sources as nexuiz --- cron-jobs/sourceballs.skip | 1 + 1 file changed, 1 insertion(+) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs.skip b/cron-jobs/sourceballs.skip index e69de29..9f11862 100644 --- a/cron-jobs/sourceballs.skip +++ b/cron-jobs/sourceballs.skip @@ -0,0 +1 @@ +nexuiz-data -- cgit v1.2.3-2-g168b From 44a15304da96944f440d2dce3d21111ba4e89ba4 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Nov 2010 15:24:38 +0100 Subject: sourceballs.skip: torcs-data has the same sources as torcs --- cron-jobs/sourceballs.skip | 1 + 1 file changed, 1 insertion(+) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs.skip b/cron-jobs/sourceballs.skip index 9f11862..64aa8d0 100644 --- a/cron-jobs/sourceballs.skip +++ b/cron-jobs/sourceballs.skip @@ -1 +1,2 @@ nexuiz-data +torcs-data -- cgit v1.2.3-2-g168b From d3a7d0e50748b9ac973ce24a2920f923c13694e4 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Nov 2010 15:28:34 +0100 Subject: sourceballs.skip: tremulous-data has the same sources as tremulous --- cron-jobs/sourceballs.skip | 1 + 1 file changed, 1 insertion(+) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs.skip b/cron-jobs/sourceballs.skip index 64aa8d0..b723e47 100644 --- a/cron-jobs/sourceballs.skip +++ b/cron-jobs/sourceballs.skip @@ -1,2 +1,3 @@ nexuiz-data torcs-data +tremulous-data -- cgit v1.2.3-2-g168b From 2e09ce0c56d27febc6b351e87ca00617217fb7b4 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Nov 2010 15:31:05 +0100 Subject: sourceballs.skip: ufoai-data has the same sources as ufoai --- cron-jobs/sourceballs.skip | 1 + 1 file changed, 1 insertion(+) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs.skip b/cron-jobs/sourceballs.skip index b723e47..17f3e78 100644 --- a/cron-jobs/sourceballs.skip +++ b/cron-jobs/sourceballs.skip @@ -1,3 +1,4 @@ nexuiz-data torcs-data tremulous-data +ufoai-data -- cgit v1.2.3-2-g168b From ef593548773265e0418f4a51a0ea817850dfa0ed Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Nov 2010 15:43:25 +0100 Subject: Blacklist more duplicate data packages --- cron-jobs/sourceballs.skip | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs.skip b/cron-jobs/sourceballs.skip index 17f3e78..ed88f85 100644 --- a/cron-jobs/sourceballs.skip +++ b/cron-jobs/sourceballs.skip @@ -2,3 +2,8 @@ nexuiz-data torcs-data tremulous-data ufoai-data +frogatto-data +vdrift-data +naev-data +btanks-data +wesnoth-data -- cgit v1.2.3-2-g168b From 027e0f547c445cce252c8cccfe3d5342de9bb5ec Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Nov 2010 17:02:09 +0100 Subject: sourceballs: Of course sort does only operate on lines --- cron-jobs/sourceballs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 23e8421..b708009 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -22,7 +22,7 @@ for repo in ${PKGREPOS[@]}; do # Read packages from db file # Format is: /- pkgs=($(bsdtar -xOf "${dbfile}" \ - | awk '/^%NAME%/{getline b};/^%BASE%/{getline b};/^%VERSION%/{getline v};/^%ARCH%/{printf "%s/%s ", b, v}' \ + | awk '/^%NAME%/{getline b};/^%BASE%/{getline b};/^%VERSION%/{getline v};/^%ARCH%/{printf "%s/%s\n", b, v}' \ | sort -u)) repo_unlock ${repo} ${arch} -- cgit v1.2.3-2-g168b From 073f431b2717fec1a3fad24158ca6e2d885ac4f3 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Nov 2010 17:46:01 +0100 Subject: texlive sources are already on ftp.archlinux.org --- cron-jobs/sourceballs.skip | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs.skip b/cron-jobs/sourceballs.skip index ed88f85..14d6f4b 100644 --- a/cron-jobs/sourceballs.skip +++ b/cron-jobs/sourceballs.skip @@ -7,3 +7,23 @@ vdrift-data naev-data btanks-data wesnoth-data +texlive-bin +texlive-bibtexextra +texlive-core +texlive-fontsextra +texlive-formatsextra +texlive-games +texlive-genericextra +texlive-htmlxml +texlive-humanities +texlive-langcjk +texlive-langcyrillic +texlive-langextra +texlive-langgreek +texlive-latexextra +texlive-music +texlive-pictures +texlive-plainextra +texlive-pstricks +texlive-publishers +texlive-science -- cgit v1.2.3-2-g168b From 9c624b099eb16e27c80ed8ff5f551ce6e8ad0682 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Mon, 22 Nov 2010 22:21:37 +0100 Subject: check_archlinux: fix for python2 usage --- cron-jobs/check_archlinux/README | 2 +- cron-jobs/check_archlinux/alpm.so | Bin 7238 -> 7166 bytes cron-jobs/check_archlinux/check_packages.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux/README b/cron-jobs/check_archlinux/README index f3a1b90..8f178dc 100644 --- a/cron-jobs/check_archlinux/README +++ b/cron-jobs/check_archlinux/README @@ -1,5 +1,5 @@ 1) Build the python module -$ python setup.py build +$ python2 setup.py build 2) copy it back to the current working directory $ cp build/lib.*/alpm.* . diff --git a/cron-jobs/check_archlinux/alpm.so b/cron-jobs/check_archlinux/alpm.so index b87b894..50f0f97 100755 Binary files a/cron-jobs/check_archlinux/alpm.so and b/cron-jobs/check_archlinux/alpm.so differ diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index e8830e0..465d87f 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # check_archlinux.py # -- cgit v1.2.3-2-g168b From 1dca721c5149f69067f38d4a33dabbebef98009b Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Mon, 22 Nov 2010 22:41:21 +0100 Subject: Attempt to fix integrity-check --- cron-jobs/integrity-check | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index c120cfd..0a8650c 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -1,15 +1,16 @@ #!/bin/bash -basedir="$(dirname $0)" +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" + if [ $# -ne 2 ]; then - echo "usage: $(basename $0) [,,...] " >&2 - exit 1 + die "usage: $(basename $0) [,,...] " fi for arch in ${ARCHES[@]}; do - $basedir/check_archlinux/check_packages.py \ + $(dirname $0)/check_archlinux/check_packages.py \ --repos="$1" \ --abs-tree="/srv/abs/rsync/$arch,/srv/abs/rsync/any" --arch="$arch" |\ - $basedir/devlist-mailer "Integrity Check $arch: $1" "$2" + $(dirname $0)/devlist-mailer "Integrity Check $arch: $1" "$2" done -- cgit v1.2.3-2-g168b From 4d16d294f338d5e90a92acb2096bca77e9e924e7 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 23 Nov 2010 12:04:30 +0100 Subject: Rewrite sourceballs to increase performance and reliability * Decrease file stats as much as possible * Create a list of all packages and meta data only once * Create a list of available source packages only once * Create a list of expected packages only once * Combine all three scripts into one to share data and code * Use as much information from the db files as possible and avoid using svn * Avoid attempting to create the same source package twice Logic works as follows: 1) create a list of all packages 2) Check for each package if we need a src package and create one 3) During this process create a list of all src packages that should be there 4) Diff both lists for the cleanup --- cron-jobs/sourceballs | 144 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 99 insertions(+), 45 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index b708009..3f690e7 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -1,57 +1,111 @@ #!/bin/bash -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" +dirname="$(dirname $0)" +. "${dirname}/../db-functions" +. "${dirname}/../config" script_lock -set_umask +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_lock ${repo} ${arch} || exit 1 + done +done + +# Create a readable file for each repo with the following format +# - [ ] +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + warning "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} not found, skipping" + continue + fi + bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ + | awk '/^%NAME%/ { getline b }; + /^%BASE%/ { getline b }; + /^%VERSION%/ { getline v }; + /^%LICENSE%/,/^$/ { + if ( !/^%LICENSE%/ ) { l=l" "$0 } + }; + /^%ARCH%/ { + getline a; + printf "%s %s %s %s\n", b, v, a, l; + l=""; + }' + done | sort -u > "${WORKDIR}/db-${repo}" +done + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_unlock ${repo} ${arch} + done +done -dirname="$(/bin/readlink -f $(/usr/bin/dirname $0))" +# Create a list of all available source package file names +find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" +# Check for all packages if we need to build a source package for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do - ftppath="${FTP_BASE}/$repo/os/$arch" - dbfile="${ftppath}/${repo}${DBEXT}" - if [ ! -r "${dbfile}" ]; then - warning "${dbfile} not found, skipping" - continue - fi - - repo_lock ${repo} ${arch} || exit 1 - # Read packages from db file - # Format is: /- - pkgs=($(bsdtar -xOf "${dbfile}" \ - | awk '/^%NAME%/{getline b};/^%BASE%/{getline b};/^%VERSION%/{getline v};/^%ARCH%/{printf "%s/%s\n", b, v}' \ - | sort -u)) - repo_unlock ${repo} ${arch} - - for pkg in ${pkgs[@]}; do - pkgbase=${pkg%/*} - pkgver=${pkg#*/} - srcpkg="${pkgbase}-${pkgver}${SRCEXT}" - - # Don't do anything for package in this 'blacklist' - if grep -q "^${pkgbase}\$" "$dirname/sourceballs.skip"; then - continue - fi - - # Use this file to 'whitelist' or force building some sourceballs, - # skipping the license check - force="" - if grep -q "^$pkgbase\$" "$dirname/sourceballs.force"; then - force="-f" - fi - - if [ ! -f "${FTP_BASE}/${SRCPOOL}/$srcpkg" ]; then - if ! $dirname/../misc-scripts/make-sourceball $force $pkgbase $repo $arch; then - error "Failed to download sources for $pkgbase" - fi - fi - done - done + msg "Updating source packages for ${repo}..." + while read line; do + pkginfo=(${line}) + pkgbase=${pkginfo[0]} + pkgver=${pkginfo[1]} + pkgarch=${pkginfo[2]} + pkglicense=(${pkginfo[@]:3}) + + # Should this package be skipped? + if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then + continue + fi + # Check if the license or .force file does not enforce creating a source package + if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then + continue + fi + # Store the expected file name of the source package + echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" + + # Build the source package if its not already there + if ! grep -Fqx "${pkgbase}-${pkgver}${SRCEXT}" "${WORKDIR}/available-src-pkgs"; then + msg2 "${pkgbase}" + # Looks like a previous arch faild; skip it + if [ -d "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" ]; then + continue + fi + mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" + svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ + "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null + if [ $? -ge 1 ]; then + error "Could not check out ${pkgbase}/repos/${repo}-${pkgarch}" + continue + fi + + pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null + makepkg --nocolor --allsource --ignorearch >/dev/null 2>&1 + if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then + mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" + # Avoid creating the same source package for every arch + echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" + else + error "Could not create source package for ${pkgbase}/repos/${repo}-${pkgarch}" + fi + popd >/dev/null + fi + done < "${WORKDIR}/db-${repo}" done -$dirname/../misc-scripts/sourceballs-cleanup +# Cleanup old source packages +cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" +cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort" +old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) + +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages..." + ${CLEANUP_DRYRUN} && warning 'dry run mode is active' + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + ${CLEANUP_DRYRUN} || mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + done +fi script_unlock -- cgit v1.2.3-2-g168b From 68a93c32579aae3ea47c11dc245d5580a4444274 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 23 Nov 2010 12:57:52 +0100 Subject: sourceballs: be less verbose --- cron-jobs/sourceballs | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 3f690e7..f226288 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -16,8 +16,8 @@ done # - [ ] for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do + # Repo does not exist; skip it if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then - warning "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} not found, skipping" continue fi bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ @@ -46,7 +46,8 @@ find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort # Check for all packages if we need to build a source package for repo in ${PKGREPOS[@]}; do - msg "Updating source packages for ${repo}..." + newpkgs=() + failedpkgs=() while read line; do pkginfo=(${line}) pkgbase=${pkginfo[0]} @@ -67,16 +68,15 @@ for repo in ${PKGREPOS[@]}; do # Build the source package if its not already there if ! grep -Fqx "${pkgbase}-${pkgver}${SRCEXT}" "${WORKDIR}/available-src-pkgs"; then - msg2 "${pkgbase}" # Looks like a previous arch faild; skip it if [ -d "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" ]; then continue fi mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ - "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null + "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 if [ $? -ge 1 ]; then - error "Could not check out ${pkgbase}/repos/${repo}-${pkgarch}" + failedpkgs[${#failedpkgs[*]}]="${pkgbase}" continue fi @@ -87,11 +87,26 @@ for repo in ${PKGREPOS[@]}; do # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" else - error "Could not create source package for ${pkgbase}/repos/${repo}-${pkgarch}" + failedpkgs[${#failedpkgs[*]}]="${pkgbase}" fi popd >/dev/null + + newpkgs[${#newpkgs[*]}]="${pkgbase}" fi done < "${WORKDIR}/db-${repo}" + + if [ ${#newpkgs[@]} -ge 1 ]; then + msg "Adding source packages for [${repo}]..." + for new_pkg in ${newpkgs[@]}; do + msg2 "${new_pkg}" + done + fi + if [ ${#failedpkgs[@]} -ge 1 ]; then + msg "Failed to create source packages for [${repo}]..." + for failed_pkg in ${failedpkgs[@]}; do + msg2 "${failed_pkg}" + done + fi done # Cleanup old source packages -- cgit v1.2.3-2-g168b From f318028961ebf045383bab26305b9232b1d34b94 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 23 Nov 2010 13:12:16 +0100 Subject: sourceballs: fix output --- cron-jobs/sourceballs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index f226288..16c41f8 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -68,15 +68,11 @@ for repo in ${PKGREPOS[@]}; do # Build the source package if its not already there if ! grep -Fqx "${pkgbase}-${pkgver}${SRCEXT}" "${WORKDIR}/available-src-pkgs"; then - # Looks like a previous arch faild; skip it - if [ -d "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" ]; then - continue - fi mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 if [ $? -ge 1 ]; then - failedpkgs[${#failedpkgs[*]}]="${pkgbase}" + failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" continue fi @@ -86,12 +82,11 @@ for repo in ${PKGREPOS[@]}; do mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" + newpkgs[${#newpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" else - failedpkgs[${#failedpkgs[*]}]="${pkgbase}" + failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" fi popd >/dev/null - - newpkgs[${#newpkgs[*]}]="${pkgbase}" fi done < "${WORKDIR}/db-${repo}" -- cgit v1.2.3-2-g168b From 79f70daa2a450f24d7ce00957651c2cac89599ec Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 23 Nov 2010 13:19:59 +0100 Subject: sourceballs: avoid failing twice --- cron-jobs/sourceballs | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 16c41f8..9b6a1e4 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -68,6 +68,12 @@ for repo in ${PKGREPOS[@]}; do # Build the source package if its not already there if ! grep -Fqx "${pkgbase}-${pkgver}${SRCEXT}" "${WORKDIR}/available-src-pkgs"; then + # Check if we had failed before + if in_array "${pkgbase}-${pkgver}${SRCEXT}" ${failedpkgs[@]}; then + continue + fi + + # Get the sources from svn mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 @@ -76,6 +82,7 @@ for repo in ${PKGREPOS[@]}; do continue fi + # Build the actual source package pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null makepkg --nocolor --allsource --ignorearch >/dev/null 2>&1 if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then -- cgit v1.2.3-2-g168b From 2a4938ecf8e9bddc75cd2533812cb587812eb928 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 23 Nov 2010 14:44:16 +0100 Subject: sourceballs: Move to our workdir first to fix warning with sudo usage --- cron-jobs/sourceballs | 2 ++ 1 file changed, 2 insertions(+) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 9b6a1e4..9d52d11 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -1,5 +1,7 @@ #!/bin/bash +pushd "${WORKDIR}" >/dev/null + dirname="$(dirname $0)" . "${dirname}/../db-functions" . "${dirname}/../config" -- cgit v1.2.3-2-g168b From 3455be4e4a3fa1c70580fd6ccb521f61c98c64c2 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Wed, 24 Nov 2010 07:32:18 +0100 Subject: sourceballs: fix working directory --- cron-jobs/sourceballs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 9d52d11..6a86a0d 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -1,10 +1,9 @@ #!/bin/bash -pushd "${WORKDIR}" >/dev/null - -dirname="$(dirname $0)" +dirname="$(dirname $(readlink -e $0))" . "${dirname}/../db-functions" . "${dirname}/../config" +pushd "${WORKDIR}" >/dev/null script_lock -- cgit v1.2.3-2-g168b From cadd215e9d726030bd7b86396eda47c2c69c1bbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=C3=A9langer?= Date: Wed, 24 Nov 2010 15:13:13 -0500 Subject: Added seperate dryrun options for the packages and sources cleanup scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Eric Bélanger Signed-off-by: Pierre Schmitz --- cron-jobs/sourceballs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 6a86a0d..9ba003a 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -119,10 +119,10 @@ old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-s if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old source packages..." - ${CLEANUP_DRYRUN} && warning 'dry run mode is active' + ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" - ${CLEANUP_DRYRUN} || mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + ${SOURCE_CLEANUP_DRYRUN} || mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" done fi -- cgit v1.2.3-2-g168b From ec20965dbd3180a92d2daa69a075284688b1be04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=C3=A9langer?= Date: Wed, 24 Nov 2010 15:31:22 -0500 Subject: Make ftpdir-cleanup less verbose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Eric Bélanger Signed-off-by: Pierre Schmitz --- cron-jobs/ftpdir-cleanup | 1 - 1 file changed, 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 914ef8c..1df4edc 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -31,7 +31,6 @@ ${CLEANUP_DRYRUN} && warning 'dry run mode is active' for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then - warning "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} not found, skipping" continue fi # get a list of actual available package files -- cgit v1.2.3-2-g168b From 447986d17554c183d958cf7de79d1904fc32aaf6 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 25 Nov 2010 13:28:04 +0100 Subject: Add FILESEXT variable for use in create-filelists --- cron-jobs/create-filelists | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index d8a5ba1..010194b 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -18,15 +18,13 @@ MYTMPDIR="$(mktemp -d ${WORKDIR}/create-filelists.tmpdir.XXXXXX)" || exit 1 #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null -case "${DBEXT}" in +case "${FILESEXT}" in *.gz) TAR_OPT="z" ;; *.bz2) TAR_OPT="j" ;; *.xz) TAR_OPT="J" ;; - *) die "Unknown compression type for DBEXT=${DBEXT}" ;; + *) die "Unknown compression type for FILESEXT=${FILESEXT}" ;; esac -FILESEXT="${DBEXT//db/files}" - for repo in ${PKGREPOS[@]}; do REPO_DB_FILE="${repo}$DBEXT" FILES_DB_FILE="${repo}$FILESEXT" @@ -77,7 +75,7 @@ for repo in ${PKGREPOS[@]}; do mkdir -p "$pkgdir" pushd "${MYTMPDIR}/${repodir}" >/dev/null [ -f "${pkgdir}/${FILES_DB_FILE}" ] && rm "${pkgdir}/${FILES_DB_FILE}" - bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f "${pkgdir}/${FILES_DB_FILE}" * + bsdtar --exclude=*${FILESEXT} -c${TAR_OPT}f "${pkgdir}/${FILES_DB_FILE}" * popd >/dev/null fi popd >/dev/null -- cgit v1.2.3-2-g168b From e792abefa6eff4cf54616add0b19d731d486045f Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 25 Nov 2010 16:03:44 +0100 Subject: Rewrote create-filelists * use correct locking of the repos * added test case * removes file lists of deleted packages * add compression independent symlink to files db --- cron-jobs/create-filelists | 147 +++++++++++++++++++++++++-------------------- 1 file changed, 81 insertions(+), 66 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index 010194b..c3e7d72 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -3,20 +3,42 @@ . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" -reposdir=${FTP_BASE} -targetdir=${FTP_BASE} - script_lock -# location where the package DB is extracted so we know what to include -DBDIR="$(mktemp -d ${WORKDIR}/create-filelists.dbdir.XXXXXX)" || exit 1 -# location where the old files DB is extracted to save us some work -CACHEDIR="$(mktemp -d ${WORKDIR}/create-filelists.cachedir.XXXXXX)" || exit 1 -# location where the new files DB is built up and eventually zipped -MYTMPDIR="$(mktemp -d ${WORKDIR}/create-filelists.tmpdir.XXXXXX)" || exit 1 +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_lock ${repo} ${arch} || exit 1 + done +done #adjust the nice level to run at a lower priority -/usr/bin/renice +10 -p $$ > /dev/null +renice +10 -p $$ > /dev/null + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repodb="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" + filedb="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" + + if [ ! -f "${repodb}" ]; then + continue + fi + # get a list of package files defined in the repo db + mkdir -p "${WORKDIR}/db-dir-${repo}-${arch}" + bsdtar -xf "${repodb}" -C "${WORKDIR}/db-dir-${repo}-${arch}" + find "${WORKDIR}/db-dir-${repo}-${arch}" -name 'desc' \ + -exec awk '/^%FILENAME%/{getline;print}' {} \; | sort > "${WORKDIR}/db-${repo}-${arch}" + + # get a list of package files defined in the files db + mkdir -p "${WORKDIR}/files-current-dir-${repo}-${arch}" + if [ ! -f "${filedb}" ]; then + echo > "${WORKDIR}/files-${repo}-${arch}" + else + bsdtar -xf "${filedb}" -C "${WORKDIR}/files-current-dir-${repo}-${arch}" + find "${WORKDIR}/files-current-dir-${repo}-${arch}" -name 'desc' \ + -exec awk '/^%FILENAME%/{getline;print}' {} \; | sort > "${WORKDIR}/files-${repo}-${arch}" + fi + done +done case "${FILESEXT}" in *.gz) TAR_OPT="z" ;; @@ -26,62 +48,55 @@ case "${FILESEXT}" in esac for repo in ${PKGREPOS[@]}; do - REPO_DB_FILE="${repo}$DBEXT" - FILES_DB_FILE="${repo}$FILESEXT" - for arch in ${ARCHES[@]}; do - pushd "$reposdir" >/dev/null - - repodir="${repo}/os/${arch}" - cached="no" - - # extract package db archive - if [ -f "${targetdir}/${repodir}/${REPO_DB_FILE}" ]; then - mkdir -p "${DBDIR}/${repodir}" - bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${DBDIR}/${repodir}" - else - warning "The repo $repo with arch $arch does not exist" - continue - fi - - # extract old file archive - if [ -f "${targetdir}/${repodir}/${FILES_DB_FILE}" ]; then - mkdir -p "${CACHEDIR}/${repodir}" - bsdtar -xf "${targetdir}/${repodir}/${FILES_DB_FILE}" -C "${CACHEDIR}/${repodir}" - cached="yes" - fi - - # create file lists - for pkg in $(ls ${DBDIR}/${repodir}); do - dbpkgdir="${DBDIR}/${repodir}/${pkg}" - cachepkgdir="${CACHEDIR}/${repodir}/${pkg}" - tmppkgdir="${MYTMPDIR}/${repodir}/${pkg}" - mkdir -p "$tmppkgdir" - ln "${dbpkgdir}/desc" "${tmppkgdir}/desc" - ln "${dbpkgdir}/depends" "${tmppkgdir}/depends" - if [ -f "${cachepkgdir}/files" ]; then - ln "${cachepkgdir}/files" "${tmppkgdir}/files" - else - filename=$(grep -A1 '^%FILENAME%$' "${dbpkgdir}/desc" | tail -n1) - echo '%FILES%' > "${tmppkgdir}/files" - bsdtar --exclude=.* -tf "$repodir/$filename" >> "${tmppkgdir}/files" - cached="no" - fi - done - - # create new file archive - if [ "$cached" == "no" -a -d "${MYTMPDIR}/${repodir}" ]; then - # at least one package has changed, so let's rebuild the archive - pkgdir="${targetdir}/${repodir}" - mkdir -p "$pkgdir" - pushd "${MYTMPDIR}/${repodir}" >/dev/null - [ -f "${pkgdir}/${FILES_DB_FILE}" ] && rm "${pkgdir}/${FILES_DB_FILE}" - bsdtar --exclude=*${FILESEXT} -c${TAR_OPT}f "${pkgdir}/${FILES_DB_FILE}" * - popd >/dev/null - fi - popd >/dev/null - done + for arch in ${ARCHES[@]}; do + filedb="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" + + if [ ! -f "${WORKDIR}/db-${repo}-${arch}" ]; then + # remove any files db that might be in this empty repo + if [ -f "${filedb}" ]; then + rm -f "${filedb}" + fi + continue + fi + + # Check if updating the files db is needed + if ! diff -q "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}" >/dev/null; then + mkdir -p "${WORKDIR}/files-new-dir-${repo}-${arch}" + + # Include all unchanged file lists + # Note: deleted packages are implicitly excluded + for f in $(comm -12 "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}"); do + mv "${WORKDIR}/files-current-dir-${repo}-${arch}/${f%*-*${PKGEXT}}" \ + "${WORKDIR}/files-new-dir-${repo}-${arch}" + done + + # Create file lists for new packages + for f in $(comm -23 "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}"); do + tdir="${WORKDIR}/files-new-dir-${repo}-${arch}/${f%*-*${PKGEXT}}" + mkdir "${tdir}" + echo '%FILES%' > "${tdir}/files" + bsdtar --exclude=.* -tf "${FTP_BASE}/${repo}/os/${arch}/${f}" >> "${tdir}/files" + + # add desc and depends file from db + dbdir="${WORKDIR}/db-dir-${repo}-${arch}/${f%*-*${PKGEXT}}" + mv "${dbdir}/desc" "${tdir}/desc" + mv "${dbdir}/depends" "${tdir}/depends" + done + + # Create the actual file db + pushd "${WORKDIR}/files-new-dir-${repo}-${arch}" >/dev/null + bsdtar -c${TAR_OPT}f "${WORKDIR}/${arch}-${repo}${FILESEXT}" * + popd >/dev/null + mv -f "${WORKDIR}/${arch}-${repo}${FILESEXT}" "${filedb}" + ln -sf "${repo}${FILESEXT}" "${filedb%.tar.*}" + fi + done +done + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_unlock ${repo} ${arch} + done done script_unlock - -# vim: set ts=4 sw=4 et ft=sh: -- cgit v1.2.3-2-g168b From dea8091103c9683e29477e1828b088e239f414ed Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 25 Nov 2010 16:06:34 +0100 Subject: sourceballs: be nice --- cron-jobs/sourceballs | 3 +++ 1 file changed, 3 insertions(+) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 9ba003a..6553327 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -13,6 +13,9 @@ for repo in ${PKGREPOS[@]}; do done done +#adjust the nice level to run at a lower priority +renice +10 -p $$ > /dev/null + # Create a readable file for each repo with the following format # - [ ] for repo in ${PKGREPOS[@]}; do -- cgit v1.2.3-2-g168b From b9dea6f306df1ffb051f0c7239f416a34f93a71a Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 25 Nov 2010 18:42:19 +0100 Subject: update-web-db: Lock db and script * usage of db-functions * Lock the db and work on a copy to avoid long lock times * Lock the script itself to avoid multiple calls * Adjust to coding style to other dbscripts --- cron-jobs/update-web-db | 72 +++++++++++++++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 26 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 3f4a2fc..8760fc6 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -1,51 +1,71 @@ #!/bin/bash -# run at nice 5. it can churn quite a bit of cpu after all. -renice +5 -p $$ > /dev/null +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" # setup paths SPATH="/srv/http/archweb" -ARCHES="i686 x86_64" # having "more important repos" last should make [core] trickle to the top of # the updates list each hour rather than being overwhelmed by big [extra] and # [community] updates -REPOS="community-testing multilib-testing multilib community extra testing core" +REPOS=('community-testing' 'multilib-testing' 'multilib' 'community' 'extra' 'testing' 'core') LOGOUT="/tmp/archweb_update.log" # figure out what operation to perform cmd="$(basename $0)" if [[ $cmd != "update-web-db" && $cmd != "update-web-files-db" ]]; then - echo "Invalid command name '$cmd' specified!" - exit 1 + die "Invalid command name '$cmd' specified!" fi +script_lock + +# run at nice 5. it can churn quite a bit of cpu after all. +renice +5 -p $$ > /dev/null + echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" -## do the dirty, then output the result to a file. -cd $SPATH -for repo in ${REPOS}; do - for arch in ${ARCHES}; do - case "$cmd" in - update-web-db) - dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.db.tar.gz" - flags="" ;; - update-web-files-db) - dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.files.tar.gz" - flags="--filesonly" ;; - esac - if [[ -f $dbfile ]]; then - echo "Updating ${repo}-${arch}" >> "${LOGOUT}" - ./manage.py reporead ${flags} ${arch} ${dbfile} >> "${LOGOUT}" 2>&1 - echo "" >> "${LOGOUT}" - fi - done +case "$cmd" in + update-web-db) + dbfileext="${DBEXT}" + flags="" + ;; + update-web-files-db) + dbfileext="${FILESEXT}" + flags="--filesonly" + ;; +esac + +# Lock the repos and get a copy of the db files to work on +for repo in ${REPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_lock ${repo} ${arch} || exit 1 + dbfile="/srv/ftp/${repo}/os/${arch}/${repo}${dbfileext}" + if [ -f "${dbfile}" ]; then + mkdir -p "${WORKDIR}/${repo}/${arch}" + cp "${dbfile}" "${WORKDIR}/${repo}/${arch}/${repo}${dbfileext}" + fi + repo_unlock ${repo} ${arch} + done done +# Run reporead on our db copy +pushd $SPATH >/dev/null +for repo in ${REPOS[@]}; do + for arch in ${ARCHES[@]}; do + dbcopy="${WORKDIR}/${repo}/${arch}/${repo}${dbfileext}" + if [ -f "${dbcopy}" ]; then + echo "Updating ${repo}-${arch}" >> "${LOGOUT}" + ./manage.py reporead ${flags} ${arch} "${dbcopy}" >> "${LOGOUT}" 2>&1 + echo "" >> "${LOGOUT}" + fi + done +done +popd >/dev/null echo "" >> "${LOGOUT}" # rotate the file if it is getting big (> 10M), overwriting any old backup if [[ $(stat -c%s "${LOGOUT}") -gt 10485760 ]]; then - mv "${LOGOUT}" "${LOGOUT}.old" + mv "${LOGOUT}" "${LOGOUT}.old" fi -# vim: set ts=4 sw=4 et ft=sh: +script_unlock -- cgit v1.2.3-2-g168b From 88a00e08e0394c95f65836be043da75fa2a131f1 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 25 Nov 2010 19:05:53 +0100 Subject: create-filelists: Improve performance by reading package list from db instead from fs --- cron-jobs/create-filelists | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index c3e7d72..8bcfc0b 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -25,17 +25,16 @@ for repo in ${PKGREPOS[@]}; do # get a list of package files defined in the repo db mkdir -p "${WORKDIR}/db-dir-${repo}-${arch}" bsdtar -xf "${repodb}" -C "${WORKDIR}/db-dir-${repo}-${arch}" - find "${WORKDIR}/db-dir-${repo}-${arch}" -name 'desc' \ - -exec awk '/^%FILENAME%/{getline;print}' {} \; | sort > "${WORKDIR}/db-${repo}-${arch}" + # This should actualy be faster than reading all the just extracted files + bsdtar -xOf "${repodb}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/db-${repo}-${arch}" # get a list of package files defined in the files db mkdir -p "${WORKDIR}/files-current-dir-${repo}-${arch}" if [ ! -f "${filedb}" ]; then - echo > "${WORKDIR}/files-${repo}-${arch}" + touch "${WORKDIR}/files-${repo}-${arch}" else bsdtar -xf "${filedb}" -C "${WORKDIR}/files-current-dir-${repo}-${arch}" - find "${WORKDIR}/files-current-dir-${repo}-${arch}" -name 'desc' \ - -exec awk '/^%FILENAME%/{getline;print}' {} \; | sort > "${WORKDIR}/files-${repo}-${arch}" + bsdtar -xOf "${filedb}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/files-${repo}-${arch}" fi done done -- cgit v1.2.3-2-g168b From 639bef5d42d64871bc332c5618e29d457962d5f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Oudompheng?= Date: Sat, 4 Dec 2010 18:09:18 +0100 Subject: Style changes: remove usage of the 'has_key' method. Signed-off-by: Pierre Schmitz --- cron-jobs/check_archlinux/check_packages.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index 465d87f..3874937 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -86,7 +86,7 @@ def parse_data(repo,data): pkg.name = line pkg.repo = repo dup = None - if packages.has_key(pkg.name): + if pkg.name in packages: dup = packages[pkg.name] else: packages[pkg.name] = pkg @@ -171,11 +171,11 @@ def provcmp(pkg,dep): def verify_dep(dep): dep = splitdep(dep) - if packages.has_key(dep.name): + if dep.name in packages: pkg = packages[dep.name] if depcmp(pkg.name,pkg.version,dep): return [pkg] - if provisions.has_key(dep.name): + if dep.name in provisions: provlist = provisions[dep.name] results = [] for prov in provlist: @@ -212,12 +212,12 @@ def compute_deplist(pkg): stack = [pkg] while stack != []: dep = stack.pop() - if pkgdeps.has_key(dep): + if dep in pkgdeps: for dep2 in pkgdeps[dep]: if dep2 not in list: list.append(dep2) stack.append(dep2) - if makepkgdeps.has_key(dep): + if dep in makepkgdeps: for dep2 in makepkgdeps[dep]: if dep2 not in list: list.append(dep2) @@ -245,7 +245,7 @@ def get_repo_hierarchy(repo): repo_hierarchy = {'core': ['core'], \ 'extra': ['core', 'extra'], \ 'community': ['core', 'extra', 'community']} - if repo_hierarchy.has_key(repo): + if repo in repo_hierarchy: return repo_hierarchy[repo] else: return ['core','extra','community'] @@ -277,10 +277,10 @@ def tarjan(pkg): checked_deps.append(pkg) S.append(pkg) deps = [] - if pkgdeps.has_key(pkg): + if pkg in pkgdeps: deps = pkgdeps[pkg] for dep in deps: - if not pkgindex.has_key(dep): + if dep not in pkgindex: tarjan(dep) pkglowlink[pkg] = min(pkglowlink[pkg],pkglowlink[dep]) elif dep in S: @@ -430,7 +430,7 @@ parse_pkgbuilds(loadrepos,arch) for name,pkg in packages.iteritems(): for prov in pkg.provides: provname=prov.split("=")[0] - if not provisions.has_key(provname): + if provname not in provisions: provisions[provname] = [] provisions[provname].append(pkg) @@ -482,7 +482,7 @@ makedep_hierarchy = check_hierarchy(makedeph) print "==> checking for circular dependencies" # make sure pkgdeps is filled for every package for name,pkg in packages.iteritems(): - if not pkgdeps.has_key(pkg): + if pkg not in pkgdeps: (deps,missdeps,_) = verify_deps(name,pkg.repo,pkg.deps) pkgdeps[pkg] = deps find_scc(repopkgs.values()) @@ -490,7 +490,7 @@ find_scc(repopkgs.values()) print "==> checking for differences between db files and pkgbuilds" for repo in repos: for pkg in dbpkgs[repo]: - if not (pkg in repopkgs.keys() and repopkgs[pkg].repo == repo): + if not (pkg in repopkgs and repopkgs[pkg].repo == repo): dbonly.append("%s/%s" % (repo,pkg)) for name,pkg in repopkgs.iteritems(): if not name in dbpkgs[pkg.repo]: -- cgit v1.2.3-2-g168b From dd5b341abea441a6a3e729afc6d1231bc0b0abe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Oudompheng?= Date: Sat, 4 Dec 2010 18:19:45 +0100 Subject: Style changes: add parentheses around print "...". Signed-off-by: Pierre Schmitz --- cron-jobs/check_archlinux/check_packages.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index 3874937..948a61a 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -421,9 +421,9 @@ for repo in repos: loadrepos = loadrepos | set(get_repo_hierarchy(repo)) print_heading("Integrity Check " + arch + " of " + ",".join(repos)) -print "\nPerforming integrity checks..." +print("\nPerforming integrity checks...") -print "==> parsing pkgbuilds" +print("==> parsing pkgbuilds") parse_pkgbuilds(loadrepos,arch) # fill provisions @@ -439,16 +439,16 @@ for name,pkg in packages.iteritems(): if pkg.repo in repos: repopkgs[name] = pkg -print "==> parsing db files" +print("==> parsing db files") dbpkgs = parse_dbs(repos,arch) -print "==> checking mismatches" +print("==> checking mismatches") for name,pkg in repopkgs.iteritems(): pkgdirname = pkg.path.split("/")[-1] if name != pkgdirname and pkg.base != pkgdirname: mismatches.append(name + " vs. " + pkg.path) -print "==> checking archs" +print("==> checking archs") for name,pkg in repopkgs.iteritems(): archs = verify_archs(name,pkg.repo,pkg.archs) invalid_archs.extend(archs) @@ -461,25 +461,25 @@ for name,pkg in packages.iteritems(): deph,makedeph = [],[] -print "==> checking dependencies" +print("==> checking dependencies") for name,pkg in repopkgs.iteritems(): (deps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.deps) pkgdeps[pkg] = deps missing_deps.extend(missdeps) deph.extend(hierarchy) -print "==> checking makedepends" +print("==> checking makedepends") for name,pkg in repopkgs.iteritems(): (makedeps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.makedeps) makepkgdeps[pkg] = makedeps missing_makedeps.extend(missdeps) makedeph.extend(hierarchy) -print "==> checking hierarchy" +print("==> checking hierarchy") dep_hierarchy = check_hierarchy(deph) makedep_hierarchy = check_hierarchy(makedeph) -print "==> checking for circular dependencies" +print("==> checking for circular dependencies") # make sure pkgdeps is filled for every package for name,pkg in packages.iteritems(): if pkg not in pkgdeps: @@ -487,7 +487,7 @@ for name,pkg in packages.iteritems(): pkgdeps[pkg] = deps find_scc(repopkgs.values()) -print "==> checking for differences between db files and pkgbuilds" +print("==> checking for differences between db files and pkgbuilds") for repo in repos: for pkg in dbpkgs[repo]: if not (pkg in repopkgs and repopkgs[pkg].repo == repo): -- cgit v1.2.3-2-g168b From d0222cb73c50fd23abd572e080b5b5231cb6d23b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Oudompheng?= Date: Sat, 4 Dec 2010 18:20:39 +0100 Subject: Add support for the [multilib] repo. Signed-off-by: Pierre Schmitz --- cron-jobs/check_archlinux/check_packages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index 948a61a..86494b7 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -244,7 +244,8 @@ def check_hierarchy(deph): def get_repo_hierarchy(repo): repo_hierarchy = {'core': ['core'], \ 'extra': ['core', 'extra'], \ - 'community': ['core', 'extra', 'community']} + 'community': ['core', 'extra', 'community'], \ + 'multilib': ['core', 'extra', 'community', 'multilib'] } if repo in repo_hierarchy: return repo_hierarchy[repo] else: -- cgit v1.2.3-2-g168b From bff3dabaf45abbc7d6bed1b3ec1c717ae678c8b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Oudompheng?= Date: Sat, 4 Dec 2010 18:37:17 +0100 Subject: Only emit a warning when a repository does not exist. Signed-off-by: Pierre Schmitz --- cron-jobs/check_archlinux/check_packages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index 86494b7..bdbbeed 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -403,8 +403,8 @@ for absroot in absroots: for repo in repos: repopath = absroot + "/" + repo if not os.path.isdir(repopath): - print "Error : the repository " + repo + " does not exist in " + absroot - sys.exit() + print("Warning : the repository " + repo + " does not exist in " + absroot) + if not os.path.isdir(repodir): print "Error: the repository directory %s does not exist" % repodir sys.exit() -- cgit v1.2.3-2-g168b From 10116f6f0d5a0fe36b8582ed23f193683e4e5523 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 9 Dec 2010 08:28:39 +0100 Subject: Call integrity-check with specific repo list for each architecture --- cron-jobs/integrity-check | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index 0a8650c..d4f9694 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -1,16 +1,32 @@ #!/bin/bash -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" +dirname="$(dirname $0)" +. "${dirname}/../db-functions" +. "${dirname}/../config" -if [ $# -ne 2 ]; then - die "usage: $(basename $0) [,,...] " +script_lock + +if [ $# -ne 1 ]; then + die "usage: $(basename $0) " fi +mailto=$1 + +check() { + ${dirname}/check_archlinux/check_packages.py \ + --repos="${repos}" \ + --abs-tree="/srv/abs/rsync/${arch},/srv/abs/rsync/any" \ + --repo-dir="${FTP_BASE}" \ + --arch="${arch}" \ + 2>&1 | ${dirname}/devlist-mailer "Integrity Check ${arch}: ${repos}" "${mailto}" +} + +repos='core,extra,community' +arch='i686' +check + +repos='core,extra,community,multilib' +arch='x86_64' +check -for arch in ${ARCHES[@]}; do - $(dirname $0)/check_archlinux/check_packages.py \ - --repos="$1" \ - --abs-tree="/srv/abs/rsync/$arch,/srv/abs/rsync/any" --arch="$arch" |\ - $(dirname $0)/devlist-mailer "Integrity Check $arch: $1" "$2" -done +script_unlock -- cgit v1.2.3-2-g168b From 76256c33bb6fa06b79bdab6615f2da587cd80f31 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 9 Dec 2010 08:34:10 +0100 Subject: check_packages.py: remove obsolete klibc workaround --- cron-jobs/check_archlinux/check_packages.py | 6 ------ 1 file changed, 6 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index bdbbeed..0a2b10e 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -454,12 +454,6 @@ for name,pkg in repopkgs.iteritems(): archs = verify_archs(name,pkg.repo,pkg.archs) invalid_archs.extend(archs) -# ugly hack to strip the weird kblic- deps -for name,pkg in packages.iteritems(): - p = re.compile('klibc-[\w\-]{27}|klibc-\*') - pkg.deps = [dep for dep in pkg.deps if not p.match(dep)] - pkg.makedeps = [dep for dep in pkg.makedeps if not p.match(dep)] - deph,makedeph = [],[] print("==> checking dependencies") -- cgit v1.2.3-2-g168b From c736d21ae97cbd9e4f6b036029426f77f14fd063 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 18 Dec 2010 01:50:17 +0100 Subject: Touch old packages after moving them to the cleanup dirs This updates the mtime of those files and will help us to remove them later. --- cron-jobs/ftpdir-cleanup | 1 + cron-jobs/sourceballs | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 1df4edc..fe49706 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -13,6 +13,7 @@ clean_pkg() { rm -f "$pkg" else mv -f "$pkg" "$CLEANUP_DESTDIR" + touch "${CLEANUP_DESTDIR}/$(basename ${pkg})" fi done fi diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 6553327..85221a7 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -125,7 +125,10 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" - ${SOURCE_CLEANUP_DRYRUN} || mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + if ! ${SOURCE_CLEANUP_DRYRUN}; then + mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + fi done fi -- cgit v1.2.3-2-g168b From 1f81b42ce2cf0c7517609bd6d5fa941a4bc01233 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 18 Dec 2010 12:41:34 +0100 Subject: Remove old files from the cleanup dirs --- cron-jobs/ftpdir-cleanup | 8 ++++++++ cron-jobs/sourceballs | 9 +++++++++ 2 files changed, 17 insertions(+) (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index fe49706..bd2d340 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -95,6 +95,14 @@ for repo in ${PKGREPOS[@]}; do fi done +old_pkgs=($(find ${CLEANUP_DESTDIR} -type f -mtime +${CLEANUP_KEEP} -printf '%f\n')) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old packages from the cleanup directory..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + ${CLEANUP_DRYRUN} || rm -f "${CLEANUP_DESTDIR}/${old_pkg}" + done +fi for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 85221a7..4048e2f 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -132,4 +132,13 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then done fi +old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages from the cleanup directory..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + ${SOURCE_CLEANUP_DRYRUN} || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + done +fi + script_unlock -- cgit v1.2.3-2-g168b From 7519447709e1ac9a5b5edaa04b32c03ac09ee2ef Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 8 Jan 2011 12:02:11 -0600 Subject: archweb update: source virtualenv if it exists Signed-off-by: Dan McGee --- cron-jobs/update-web-db | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'cron-jobs') diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 8760fc6..6ced4c1 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -5,6 +5,8 @@ # setup paths SPATH="/srv/http/archweb" +ENVPATH="/srv/http/archweb-env/bin/activate" + # having "more important repos" last should make [core] trickle to the top of # the updates list each hour rather than being overwhelmed by big [extra] and # [community] updates @@ -24,6 +26,11 @@ renice +5 -p $$ > /dev/null echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" +# source our virtualenv if it exists +if [ -f "$ENVPATH" ]; then + . "$ENVPATH" +fi + case "$cmd" in update-web-db) dbfileext="${DBEXT}" -- cgit v1.2.3-2-g168b From d9aad61303816aa7a4b1ba950856b30368acb968 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 30 Jan 2011 12:07:30 +0100 Subject: Make sure to only remove package files from the cleanup dirs --- cron-jobs/ftpdir-cleanup | 2 +- cron-jobs/sourceballs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index bd2d340..09e8a49 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -95,7 +95,7 @@ for repo in ${PKGREPOS[@]}; do fi done -old_pkgs=($(find ${CLEANUP_DESTDIR} -type f -mtime +${CLEANUP_KEEP} -printf '%f\n')) +old_pkgs=($(find ${CLEANUP_DESTDIR} -type f -name "*${PKGEXT}" -mtime +${CLEANUP_KEEP} -printf '%f\n')) if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old packages from the cleanup directory..." for old_pkg in ${old_pkgs[@]}; do diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 4048e2f..b55de05 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -132,7 +132,7 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then done fi -old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) +old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -name "*${SRCEXT}" -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old source packages from the cleanup directory..." for old_pkg in ${old_pkgs[@]}; do -- cgit v1.2.3-2-g168b From 439bd2112c01bc9b185d2552aa0888d7055929e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Fri, 18 Feb 2011 20:43:56 -0300 Subject: First stab at sourceballing everything --- cron-jobs/sourceballs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index b55de05..0df007e 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -63,10 +63,11 @@ for repo in ${PKGREPOS[@]}; do if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then continue fi + # Commenting out, we'll sourceball everything # Check if the license or .force file does not enforce creating a source package - if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then - continue - fi +# if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then +# continue +# fi # Store the expected file name of the source package echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" @@ -79,8 +80,10 @@ for repo in ${PKGREPOS[@]}; do # Get the sources from svn mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" - svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ - "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 + #svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ + # "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 + cp -r "${SVNREPO}/libre/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ + cp -r "${SVNREPO}/libre-testing/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 if [ $? -ge 1 ]; then failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" continue -- cgit v1.2.3-2-g168b From f8c49cc2915f66b53bd9dd248b41ab6a96c1059a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sun, 20 Feb 2011 22:09:28 -0300 Subject: Changes to sourceball everything --- cron-jobs/sourceballs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 0df007e..5726484 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -21,10 +21,10 @@ renice +10 -p $$ > /dev/null for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do # Repo does not exist; skip it - if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + if [ ! -f "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue fi - bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ + bsdtar -xOf "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ | awk '/^%NAME%/ { getline b }; /^%BASE%/ { getline b }; /^%VERSION%/ { getline v }; @@ -46,7 +46,7 @@ for repo in ${PKGREPOS[@]}; do done # Create a list of all available source package file names -find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" +find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" # Check for all packages if we need to build a source package for repo in ${PKGREPOS[@]}; do @@ -59,7 +59,7 @@ for repo in ${PKGREPOS[@]}; do pkgarch=${pkginfo[2]} pkglicense=(${pkginfo[@]:3}) - # Should this package be skipped? + # Should this packages be skipped? if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then continue fi @@ -82,6 +82,9 @@ for repo in ${PKGREPOS[@]}; do mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" #svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ # "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 + + # If it's on official repos, nor [libre], nor [libre-testing] + cp -r "${SVNREPO}/$repo/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ cp -r "${SVNREPO}/libre/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ cp -r "${SVNREPO}/libre-testing/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 if [ $? -ge 1 ]; then @@ -93,7 +96,7 @@ for repo in ${PKGREPOS[@]}; do pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null makepkg --nocolor --allsource --ignorearch >/dev/null 2>&1 if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then - mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" + mv "${pkgbase}-${pkgver}${SRCEXT}" "${ARCH_BASE}/${SRCPOOL}" # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" newpkgs[${#newpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" @@ -129,7 +132,7 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" if ! ${SOURCE_CLEANUP_DRYRUN}; then - mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" fi done -- cgit v1.2.3-2-g168b From c6516eca2803b780a01a6b2bf3e081c8bd570301 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 20 Feb 2011 17:10:40 -0800 Subject: sourceballs --- cron-jobs/sourceballs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 0df007e..53d43ce 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -21,10 +21,10 @@ renice +10 -p $$ > /dev/null for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do # Repo does not exist; skip it - if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + if [ ! -f "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue fi - bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ + bsdtar -xOf "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ | awk '/^%NAME%/ { getline b }; /^%BASE%/ { getline b }; /^%VERSION%/ { getline v }; @@ -46,7 +46,7 @@ for repo in ${PKGREPOS[@]}; do done # Create a list of all available source package file names -find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" +find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" # Check for all packages if we need to build a source package for repo in ${PKGREPOS[@]}; do @@ -91,9 +91,9 @@ for repo in ${PKGREPOS[@]}; do # Build the actual source package pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null - makepkg --nocolor --allsource --ignorearch >/dev/null 2>&1 + makepkg --nocolor --allsource --ignorearch # >/dev/null 2>&1 if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then - mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" + mv "${pkgbase}-${pkgver}${SRCEXT}" "${ARCH_BASE}/${SRCPOOL}" # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" newpkgs[${#newpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" @@ -129,7 +129,7 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" if ! ${SOURCE_CLEANUP_DRYRUN}; then - mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" fi done -- cgit v1.2.3-2-g168b From 229a9c504cbd733c93cf91399dc54bedf5160cc5 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 6 Mar 2011 15:22:21 -0800 Subject: Fixes --- cron-jobs/sourceballs2 | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100755 cron-jobs/sourceballs2 (limited to 'cron-jobs') diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 new file mode 100755 index 0000000..452208e --- /dev/null +++ b/cron-jobs/sourceballs2 @@ -0,0 +1,87 @@ +#!/bin/bash + +dirname="$(dirname $(readlink -e $0))" +. "${dirname}/../db-functions" +. "${dirname}/../config" +. "${MAKEPKGCONF}" + +pushd "${WORKDIR}" >/dev/null + +script_lock + +#adjust the nice level to run at a lower priority +renice +10 -p $$ > /dev/null + +# Create a list of all available source package file names +find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" + +# Steps +# Traverse the ABSLibre +# Makepkg --allsource every package +# Remove the old packages +pushd "${SVNREPO}" >/dev/null + +failedpkgs=() +for repo in ${PKGREPOS[@]}; do + pushd $repo >/dev/null + find . -maxdepth 1 -type d | while read pkg; do + pushd "${SVNREPO}/$repo/$pkg" >/dev/null + + [[ ! -e PKGBUILD ]] && { + warning "$repo/$pkg is not a package" + continue + } + + unset pkgbase pkgname + source PKGBUILD + pkgbase=${pkgbase:-$pkgname} + + echo "${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" + + # Skip already sourceballed + [[ -e "${SRCPKGDEST}/${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" ]] && \ + continue + + makepkg --allsource --ignorearch -c >/dev/null 2>&1 + + [[ $? -ne 0 ]] && \ + failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" + + done + popd >/dev/null +done + +# Cleanup old source packages +cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" +cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort" +old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) + +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages..." + ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + if ! ${SOURCE_CLEANUP_DRYRUN}; then + mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + fi + done +fi + +old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -name "*${SRCEXT}" -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages from the cleanup directory..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + ${SOURCE_CLEANUP_DRYRUN} || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + done +fi + +msg "Failed" +for _fail in ${failedpkgs[@]}; do + msg2 "$_fail" +done + + +script_unlock + -- cgit v1.2.3-2-g168b