From fcff5c5c4341e84f6580289dff270f1af8ea7080 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernandez Date: Fri, 20 May 2011 05:48:23 -0500 Subject: Fullpkg added features. Needs debug --- fullpkg | 386 ++++++++++++++++++++++++++++++++++------------------------------ 1 file changed, 204 insertions(+), 182 deletions(-) diff --git a/fullpkg b/fullpkg index d5c7468..a428440 100755 --- a/fullpkg +++ b/fullpkg @@ -1,15 +1,16 @@ #!/bin/bash # TODO -# * Do version checking -# * Detect circular builds # * Detect pkgnames by provides, replaces, etc. instead of dir tree -source /etc/makepkg.conf -source /etc/abs.conf -source /etc/libretools.conf +eval $(egrep "PACKAGER|CARCH" /etc/makepkg.conf) +eval $(egrep "ABSROOT" /etc/abs.conf) +eval $(egrep "REPOS|source" /etc/libretools.conf) +source /usr/bin/libremessages -[[ -r ~/.config/libretools/libretools.conf ]] && \ - source ~/.config/libretools/libretools.conf +[[ -r $XDG_CONFIG_HOME/libretools/libretools.conf ]] && \ + source $XDG_CONFIG_HOME/libretools/libretools.conf + +## START FUNCTIONS ## function usage { echo "cd to a dir containing a PKGBUILD and run:" @@ -19,42 +20,13 @@ function usage { echo echo "OPTIONS:" echo " -h : this message." - echo " -f : build even when a package has been built." - echo " -n absdir : set ABSROOT to this dir" - echo " -r reponame : set repo name to reponame" - echo " -R pkgname : build pkgname if it is a dep" - echo -} - -force_build='n' -force_array=() -_fullpkgargs="" -failed=() -missing=() - -while getopts 'hfn:r:R:' arg; do - case $arg in - h) usage; exit 0 ;; - f) force_build='y' ;; - R) force_array=(${force_array[@]} $OPTARG); _fullpkgargs+="-R $OPTARG ";; - n) ABSROOT="$OPTARG" ;; - r) repo="$OPTARG" ;; - esac -done - -[[ ! -r PKGBUILD ]] && { - error "This isn't a build directory" + echo " -c : check deps" + # printf " -f pkgname : build even when a package has been built. " + # printf " Use it as many times as needed\n" + echo " -a absdir : set ABSROOT to this dir" echo - usage - exit 1 } -tmp_dir=$(mktemp -d /tmp/$(basename $PWD).XXXXXX) -queue_file=$(mktemp /tmp/queue.XXXXXX) -ban_file=$(mktemp /tmp/ban.XXXXXX) - -## START FUNCTIONS ## - # Queue Management # * Always get the queue list from the server # * Add/Remove from queue @@ -65,7 +37,7 @@ ban_file=$(mktemp /tmp/ban.XXXXXX) # Get the queue list from the server get_queue() { - rsync -e ssh -aq $PARABOLAHOST:mips64el/queue $queue_file >/dev/null 2>&1 || { + wget -N http://repo.parabolangunlinux.org/files/queue $queue_file >/dev/null 2>&1 || { error "Failed to retrieve queue list" return 1 } @@ -82,9 +54,7 @@ put_queue() { # Add packages to the queue update_queue() { get_queue || return $? - basename $PWD | sed "s/$/:$PACKAGER/" >> $queue_file || return 2 - put_queue || return $? } @@ -95,14 +65,14 @@ remove_queue() { grep -vw "^$(basename $PWD)" $queue_file > $queue_file.2 cat $queue_file.2 > $queue_file - put_queue && rm $queue_file{,.2} && return 0 || return $? + put_queue && rm $queue_file.2 && return 0 || return $? } # Checks if a package is listed check_queue() { get_queue || return $? - packager=$(grep -w "$(basename $PWD)" ${queue_file} | cut -d ':' -f2) + local packager=$(grep -w "$(basename $PWD)" ${queue_file} | cut -d ':' -f2) [[ ! -z $packager ]] && [[ "$packager" != "$PACKAGER" ]] && { warning "$(basename $PWD) is being packaged by $packager. Please wait." @@ -114,162 +84,214 @@ check_queue() { # END Queue Management # -# Checks if the package is banned from building -is_banned() { - rsync -e ssh -aq $PARABOLAHOST:mips64el/ban $ban_file >/dev/null 2>&1 || { - plain "Failed to get ban list" - return 1 - } - grep -w $1 $ban_file >/dev/null 2>&1 - return $? -} - guess_repo() { basename $(dirname $(pwd)) } -# usage : in_array( $needle, $haystack ) -# return : 0 - found -# 1 - not found -function in_array { - [[ $2 ]] || return 1 - local needle=$1; shift - local item - for item in "$@"; do - [[ ${item#@} = $needle ]] && return 0 - done - return 1 # Not Found +# Usage: cleanup [ $(basename $PWD) ] from PKGBUILD dir +# cleans the tempdir +function cleanup { + if [ ! -d $tempdir/ ]; then + return 1 + elif [ -d $tempdir -a ${#@} -gt 0 ]; then + for _dir in $@; do + rm -rf $tempdir/$_dir/ + done + fi } +# Usage: find_deps +# no parameters +function find_deps { + [ $level -lt 20 ] || return 20 + ## Check this level. + source PKGBUILD + local repo=${repo:-$(guess_repo)} + + # TODO: If this package is in force_build: skip this step + # If package is built exit + if is_built "${pkgbase:-${pkgname[0]}}>=${pkgver}-${pkgrel}"; + then + msg2 "${pkgbase:-${pkgname[0]}}>=${pkgver}-${pkgrel} : built"; exit 0 + fi + + msg2 "${pkgbase:-${pkgname[0]}}>=${pkgver}-${pkgrel}" -function quit { - remove_queue - exit 1 -} + # Tell which packages are deeper in deps (even if they are on tempdir) + # so we can build them first. + echo "${level}: $(basename $PWD)" >> "${tempdir}/.BUILDORDER" -function cleanup { - rm $ban_file $queue_file - rm -rf $tmp_dir -} + # if pkgbuild directory is on tempdir, do not copy and exit + if [ -d "${tempdir}/$(basename $PWD)" ]; then + exit 0 + else + cp -r ../$(basename $PWD) ${tempdir}/ + # Info to eval later + echo "repo=$repo" > "${tempdir}/$(basename $PWD)/.INFO" + fi -# TODO keep track of spawned fullpkgs + ## Check next levels + # Clean version checking + deps=$(echo "${depends[@]} ${makedepends[@]} ${pkgdeps[@]}" | \ + sed "s/[=<>]\+[^ ]\+//g" | \ + tr ' ' "\n" | \ + sort -u) + declare -i next_level=$level+1 + for _dep in ${deps[@]}; do + for _repo in ${REPOS[@]}; do + # try to find $_dep on each repo from dirname + [ -e "$ABSROOT/${_repo}/$_dep/PKGBUILD" ] && { + pushd "$ABSROOT/${_repo}/$_dep" >/dev/null + "$0 -c '${check_deps_file}' -d '${tempdir}' -l '${next_level}'" + # Circular deps must fail + [ $? -eq 20 ] && return 20 + popd >/dev/null + # found, go to next dep + break 1 + } + # search pkgname in repo if that doesn't work + # this should find pkgsplits + _dir=$(find "$ABSROOT/${_repo}" -type f -name PKGBUILD -print0 | \ + "xargs" -0 -e grep -H -E "pkgname|pkgbase" | grep $_dep) && { + pushd $(dirname $(echo $_dir | cut -d: -f1)) >/dev/null + "$0 -c '${check_deps_file}' -d '${tempdir}' -l '${next_level}'" + # Circular deps must fail + [ $? -eq 20 ] && return 20 + popd > /dev/null + # found, go to next dep + break 1 + } + done + done + unset next_level dir +} ## END FUNCTIONS ## -source PKGBUILD -repo=${repo:-$(guess_repo)} -msg "Building ${repo:-missing repo}/${pkgbase:-${pkgname[@]}}: $pkgdesc" +force_build="" +level=0 +while getopts 'ha:c:d:l:' arg; do + case $arg in + h) usage; exit 0 ;; + a) ABSROOT="$OPTARG" ;; + c) check_deps_file="$OPTARG" ;; + # f) force_build+="-f pkgname " ;; + d) tempdir="$OPTARG" ;; + # hidden function to know what to build first. + # if $level > 0 it will not build + l) level=$OPTARG ;; + esac +done +mkdir -p $XDG_CONFIG_HOME/libretools +queue_file=$XDG_CONFIG_HOME/libretools/queue +ban_file=$XDG_CONFIG_HOME/libretools/ban +touch $queue_file $ban_file +# Only on level 0 +[ $level -eq 0 ] && { + # if tempdir exist use it, else make a tempdir + tempdir="$(mktemp -d /tmp/fullpkg.XXXXXX)" + check_deps_file=$tempdir/deps + msg "Updating pacman db and packages" + sudo pacman -Syu --noconfirm + msg "Checking dependencies" +} +buildorder=$tempdir/.BUILDORDER +touch $buildorder $tempdir/deps $tempdir/false $tempdir/install +touch $check_deps_file + +[[ ! -r PKGBUILD ]] && { + error "This isn't a build directory" + usage && exit 1 +} -# Pre build tests -if [ $force_build == 'n' ]; then +## if $level = 20 there is highly likely there are circular deps +[ $level -eq 20 ] && exit 20 + +find_deps || { + # if find_deps finds circular deps + # it should exit with status 20 + [ $? -eq 20 ] && \ + # only show message on level 0 + [ $level -eq 0 ] && \ + error "Check for circular deps on $tempdir/.BUILDORDER"; + exit 20 +} - # Be able to write files - if [[ ! -w $queue_file ]]; then - error "can't write queue file" - exit 1 - elif [[ ! -w $ban_file ]] ; then - error "can't write ban file" - exit 1 +# levels greater than 0 must only check deps +[ $level -gt 0 ] && exit 0 + + +# check .BUILDORDER to not include banned deps and +[ $level -eq 0 -a -d $tempdir ] && { + # Check for banned deps + if [ -w $ban_file -a -r $ban_file ]; then + chmod o+rw $ban_file || error "Ban file is not readable/writable ($ban_file)" + else + wget -N http://repo.parabolagnulinux.org/files/ban $ban_file >/dev/null 2>&1 || \ + warning "Failed to get ban list" && [ -r $ban_file ] && { + # continue if download failed but local copy + search=$(cat $ban_file | tr "\n" "|") + echo ${@} | tr " " "\n" | egrep -w $search $buildorder | cleanup + echo ${@} | tr " " "\n" | egrep -vw $search $buildorder > $buildorder.2 + mv $buildorder.2 $buildorder + unset search + } fi +} - if is_built "${pkgbase:-${pkgname[0]}}>=${pkgver}-${pkgrel}"; then - msg2 "This package is built." - exit 0 +## START Building +msg "Building packages:" +pushd "$tempdir" + +[ ! -w $queue_file ] && error "can't write queue file" + +trap "break" INT +trap "remove_queue" EXIT INT QUIT TERM KILL +build_packages=$(sort -gr $buildorder | cut -d: -f2) +while [ ${#build_packages[@]} -gt 0 ]; do + build_packages=$(sort -gr $buildorder | cut -d: -f2) + pushd $temp_dir/${build_packages[0]} + egrep -vw ${build_packages[0]} $buildorder > $buildorder.2 + mv $buildorder.2 $buildorder + unset build_packages + eval $(egrep "pkgname|pkgbase|pkgver|pkgrel" PKGBUILD) + msg2 "${pkgbase:-${pkgname[0]}} $pkgver-$pkgrel" + if ! grep mips64el PKGBUILD >/dev/null; then + plain "Adding mips64el arch" + sed -i "s/^\(arch=([^)anym]\+\))/\1 'mips64el')/" "PKGBUILD" fi + # Let everybody know we're building this. + update_queue || { + warning "Couldn't update the queue, let your partners know about this." + } + makepkg --noconfirm --nocheck -smciL; r=$? + case $r in + 0) plain "The build was succesful." + source .INFO && [ -n $repo ] && librestage $repo || \ + echo "unstaged: $(basename $PWD)" >> $tempdir/failed + echo "$(basename $PWD)" >> $tempdir/install + cleanup "$(basename $PWD)" + msg2 "Sync db again" + sudo pacman -Sy ;; + 1) error "There were errors while trying to build the package." + echo "failed: $(basename $PWD)" >> $tempdir/failed ;; + 2) error "The build failed." && echo "failed: $(basename $PWD)" >> $tempdir/failed ;; + esac + popd >/dev/null +done - if is_banned ${pkgbase:-$pkgname}; then - error "This package is banned from building. Check the ban list" - exit 1 - fi +msg "Those packages were installed" +plain $(cat $tempdir/install | tr "\n" " ") - check_queue || exit 1 - -fi - -# This will be executed at exit for any reason. -trap "quit" EXIT INT QUIT TERM KILL HUP - -if ! grep mips64el PKGBUILD >/dev/null; then - msg "Adding mips64el arch" - sed -i "s/^\(arch=([^)anym]\+\))/\1 'mips64el')/" "PKGBUILD" -fi - -# Clean version checking -deps=$(echo "${depends[@]} ${makedepends[@]} ${pkgdeps[@]}" | \ - sed "s/[=<>]\+[^ ]\+//g" | \ - tr ' ' "\n" | \ - sort -u) - -msg "Checking dependencies" -for _dep in ${deps[@]}; do - is_banned $_dep && continue - - for _repo in ${REPOS[@]}; do - # TODO find split packages - [[ -e "$ABSROOT/${_repo}/$_dep/PKGBUILD" ]] && { - source "$ABSROOT/${_repo}/$_dep/PKGBUILD" - msg2 "Checking for $_dep>=$pkgver-$pkgrel" - - if ! in_array $_dep ${force_array[@]}; then - if is_built "$_dep>=$pkgver-$pkgrel"; then - plain "this package is built" - break - fi - else - _fullpkgargs+="-f " - _fullpkgargs="$(echo $_fullpkgargs | sed s/"-R $_dep "//)" - force_array=( $(echo ${forcearray[@]} | tr " " "\n" | grep -vw "^$_dep") ) - fi - - cp -r "$ABSROOT/$_repo/$_dep" $tmp_dir/ || { - error "Can't copy $_dep to the work dir." - exit 1 - } - - # Enter the work dir and run this command in it - pushd $tmp_dir/$_dep >/dev/null - - $0 -r $_repo $_fullpkgargs - - [[ $? -ne 0 ]] && { - failed=(${failed[@]} $_dep) - } - - popd >/dev/null - } - done -done +msg "Uploading packages to the server" +librerelease -# TODO probably not elegant enough -# TODO only the last fullpkg should show this message -# and it should contain all failed pkgs -[[ ${#failed[@]} -gt 0 ]] && { - error "This packages failed to build: ${failed[@]}" - exit 1 +pkgs=$(cat $tempdir/failed | grep "failed:") && { + error "Those packages failed to build:" + plain "$(echo ${pkgs[@]} | cut -d: -f2)" } - -# Let everybody know we're building this -update_queue || { - warning "Couldn't update the queue, let your partners know about this." +pkgs=$(cat $tempdir/failed | grep "unstaged:") && { + error "Those packages couldn't be staged because of missing reponame:" + plain "$(cat $tempdir/failed | grep "unstaged:" | cut -d: -f2)" } -cp -r ../$(basename $PWD) $tmp_dir/ -pushd $tmp_dir/$(basename $PWD) >/dev/null - -msg "Syncing database" -sudo pacman -Syu --noconfirm -makepkg --noconfirm --nocheck -sLcr ; r=$? -case $r in - 0) msg "The build was succesful." - #TODO: mipsrelease should upload to a local db - mipsrelease *.pkg.tar.* - librestage $repo - librerelease - sudo pacman -Sy - # cleanup is only on succesfull build so failed can be inspected - cleanup;; - 1) error "There were errors while trying to build the package." ;; - 2) error "The build failed." ;; -esac - -exit $r - +exit 0 \ No newline at end of file -- cgit v1.2.3-2-g168b From 0b66a220ed2f208123b7b349d44060f3d678bc5a Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernandez Date: Wed, 25 May 2011 01:30:46 -0500 Subject: * Mostly working fullpkg-ng --- fullpkg-ng | 312 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ libretools.conf | 7 +- 2 files changed, 318 insertions(+), 1 deletion(-) create mode 100755 fullpkg-ng diff --git a/fullpkg-ng b/fullpkg-ng new file mode 100755 index 0000000..4159320 --- /dev/null +++ b/fullpkg-ng @@ -0,0 +1,312 @@ +#!/bin/bash +# TO TEST: (on find_deps) +# * Detect pkgnames by provides, replaces, etc. instead of dir tree + +eval "$(egrep "PACKAGER=|CARCH=" /etc/makepkg.conf)" +eval "$(egrep "ABSROOT=" /etc/abs.conf)" +eval "$(egrep "REPOS=|FULLBUILDCMD=" /etc/libretools.conf)" +source /usr/bin/libremessages + +[ -r $XDG_CONFIG_HOME/libretools/libretools.conf ] && \ + source $XDG_CONFIG_HOME/libretools/libretools.conf + +## START FUNCTIONS ## + +function usage { + echo "cd to a dir containing a PKGBUILD and run:" + echo "$0 [options]" + printf "This script will check dependencies, build them if possible " + printf "and stage the packages on it's repo." + echo + echo "OPTIONS:" + echo " -h : this message." + echo " -a absdir : set absdir as ABSROOT." + echo " -c : check deps only, do not build." + echo " -d build_dir : use this dir to build. Defaults to mktemp." + echo " -n : don't update pacman db." + echo " -m max_level : check deps until this level" + # printf " -f pkgname : build even when a package has been built. " + # printf " Use it as many times as needed\n" + echo +} + +# Queue Management +# * Always get the queue list from the server +# * Add/Remove from queue +# * Check if a package is listed + +# TODO +# * Check for concurrence # @ fauno: What do you mean? + +# Get the queue list from the server +get_queue() { + rsync -e ssh -aq $PARABOLAHOST:mips64el/queue $queue_file >/dev/null 2>&1 || { + error "Failed to retrieve queue list" + return 1 + } +} + +# Put the queue list on the server +put_queue() { + rsync -e ssh -aq $queue_file $PARABOLAHOST:mips64el/queue >/dev/null 2>&1 || { + error "Failed to put queue list" + return 1 + } +} + +# Add packages to the queue +update_queue() { + get_queue || return $? + echo "$(basename $PWD):$PACKAGER" >> $queue_file || return 2 + put_queue || return $? +} + +# Remove a package from the queue +remove_queue() { + get_queue || return $? + + grep -vw "^$(basename $PWD)" $queue_file > $queue_file.2 + mv $queue_file.2 > $queue_file + + put_queue && rm $queue_file.2 && return 0 || return $? +} + +# Checks if a package is listed +check_queue() { + get_queue || return $? + + local packager=$(grep -w "$(basename $PWD)" ${queue_file} | cut -d ':' -f2) + + [ ! -z $packager ] && [ "$packager" != "$PACKAGER" ] && { + warning "$(basename $PWD) is being packaged by $packager. Please wait." + return 1 + } + + return 0 +} + +# END Queue Management # + +guess_repo() { + basename $(dirname $(pwd)) +} + +# Usage: cleanup [ $(basename $PWD) ] from PKGBUILD dir +# cleans the build_dir +function cleanup { + if [ ! -d $build_dir ]; then + return 1 + elif [ -d $build_dir -a ${#@} -gt 0 ]; then + for _dir in $@; do + rm -rf $build_dir/$_dir/ + done + fi +} + +# Check PKGBUILD and find non built or outdated deps +# on ABSROOT which should be abslibre-misp64el +function find_deps { + ## Check this level. + source PKGBUILD + local repo=${repo:-$(guess_repo)} + + # If package is built exit + # TODO?: If this package is in force_build: skip this step + if is_built "${pkgbase:-${pkgname[0]}}>=${pkgver}-${pkgrel}"; then + exit 0 + fi + + # Tell which packages are deeper in deps (even if they are on build_dir) + # so we can build them first. + echo "${level}:$(basename $PWD)" >> "${build_dir}/BUILDORDER" + + # if pkgbuild directory is on build_dir, do not copy and exit + if [ -d "${build_dir}/$(basename $PWD)" ]; then + exit 0 + else + cp -r ../$(basename $PWD) ${build_dir}/ + # Info to eval later + echo "repo=$repo" > "${build_dir}/$(basename $PWD)/.INFO" + fi + + msg2 "${pkgbase:-${pkgname[0]}}>=${pkgver}-${pkgrel}" + + ## Check next levels + # Clean version checking + deps=$(echo "${depends[@]} ${makedepends[@]}" | \ + sed "s/[=<>]\+[^ ]\+//g" | \ + tr ' ' "\n" | \ + sort -u) + declare -i next_level=$level+1 + + for _dep in ${deps[@]}; do + for _repo in ${REPOS[@]}; do + # try to find $_dep on each repo from dirname + [ -e "$ABSROOT/${_repo}/$_dep/PKGBUILD" ] && { + pushd "$ABSROOT/${_repo}/$_dep" > /dev/null + $0 -c -d ${build_dir} -l ${next_level} + # Circular deps must fail + [ $? -eq 20 ] && return 20 + popd > /dev/null + } && break 1 # found, go to next dep + + # search pkgname in repo if that doesn't work + # this should find pkgsplits + _dir=$(find "$ABSROOT/${_repo}" -type f -name PKGBUILD -print0 | \ + "xargs" -0 -e grep -H -Ew grep $_dep > /dev/null ) && { + pushd $(dirname $(echo $_dir | cut -d: -f1)) > /dev/null + $0 -c -d ${build_dir} -l ${next_level} + # Circular deps must fail + [ $? -eq 20 ] && return 20 + popd > /dev/null + } && break 1 # found, go to next dep + done + done + unset next_level dir + # unset PKGBUILD variables + unset pkgname pkgver pkgrel epoch pkgdesc arch url license groups depends \ + makedepens checkdepends optdepends provides conflicts replaces backup \ + options install changelog source noextract md5sums build check package +} + +## END FUNCTIONS ## + +force_build="" +level=0 +noupdate='n' +check_deps_only='n' +max_level=21 +while getopts 'ha:cd:l:nm:' arg; do + case $arg in + h) usage; exit 0 ;; + a) ABSROOT="$OPTARG" ;; + c) check_deps_only='y' ;; + # f) force_build+="-f pkgname " ;; + d) build_dir="$OPTARG" ;; + # hidden option to know what to build first. + # if $level > 0 it will not build + l) level=$OPTARG ;; + n) noupdate='y';; + m) max_level=$OPTARG ;; + esac +done + +# Only on level 0 +[ $level -eq 0 ] && { + # if build_dir exist use it, else make a build_dir + build_dir=${build_dir:-$(mktemp -d /tmp/fullpkg.XXXXXX)} + + # set queue_file and ban_file + mkdir -p $XDG_CONFIG_HOME/libretools + queue_file=$XDG_CONFIG_HOME/libretools/queue + ban_file=$XDG_CONFIG_HOME/libretools/ban + touch $build_dir/{deps,log,BUILDORDER} $queue_file $ban_file + + [ $noupdate = 'n' ] && { + msg "Updating pacman db and packages" + sudo pacman -Syu --noconfirm + } + msg "Checking dependencies" +} +buildorder=$build_dir/BUILDORDER + +[ ! -r PKGBUILD ] && { + error "This isn't a build directory" + usage && exit 1 +} + +## if $level = 20 there is highly likely there are circular deps +[ $level -eq $max_level -o $level -gt $max_level ] && exit 20 + +find_deps || { + # if find_deps finds circular deps + # it should exit with status 20 + [ $? -eq 20 ] && { + # only show message on level 0 + [ $level -eq 0 ] && error "Check for circular deps on $build_dir/BUILDORDER"; + } + exit 20 +} + +# levels greater than 0 must only check deps +[ $check_deps_only = 'y' -o $level -gt 0 ] && exit 0 + +# check BUILDORDER to not include banned deps and +[ $level -eq 0 -a -d $build_dir ] && { + # Check for banned deps + if [ -w $ban_file -a -r $ban_file ]; then + chmod o+rw $ban_file || error "Ban file is not readable/writable ($ban_file)" + else + rsync -e ssh -aq $PARABOLAHOST:mips64el/ban >/dev/null 2>&1 || { + warning "Failed to get ban list" && [ -r $ban_file ] && { + # continue if download failed but local copy + search=$(cat $ban_file | tr "\n" "|") + echo ${@} | tr " " "\n" | egrep -w "$search" $buildorder >> $build_dir/banned + echo ${@} | tr " " "\n" | egrep -vw "$search" $buildorder > $buildorder.2 + mv $buildorder.2 $buildorder + unset search + } + } + fi +} + +## START Building + +msg "Building packages:" +cd ${build_dir} > /dev/null + +[ ! -w $queue_file ] && error "can't write queue file" + +# Exit loop on Ctrl+C +trap "break" INT +# Remove from queue package being built on error +trap "remove_queue" EXIT INT QUIT TERM KILL +build_packages=$(sort -gr $buildorder | cut -d: -f2) +while [ ${#build_packages[@]} -gt 0 ]; do + build_packages=$(sort -gr $buildorder | cut -d: -f2) + pushd $build_dir/${build_packages[0]} > /dev/null + source PKGBUILD + msg2 "${pkgbase:-${pkgname[0]}} $pkgver-$pkgrel" + if ! grep mips64el PKGBUILD >/dev/null; then + plain "Adding mips64el arch" + sed -i "s/^\(arch=([^)anym]\+\))/\1 'mips64el')/" "PKGBUILD" + fi + # Check if pkg is being built, if it's not let everybody know we're building this. + check_queue && update_queue || { + warning "Couldn't update the queue, let your partners know about this." + } + $FULLBUILDCMD; r=$? + case $r in + 0) plain "The build was succesful." + source .INFO && [ -n $repo ] && librestage $repo || \ + echo "unstaged:$(basename $PWD)" >> $build_dir/log + echo "built:$(basename $PWD)" >> $build_dir/log + cleanup "$(basename $PWD)" + ;; + 1) error "There were errors while trying to build the package." + echo "failed: $(basename $PWD)" >> $build_dir/log ;; + 2) error "The build failed." + echo "failed: $(basename $PWD)" >> $build_dir/log ;; + esac + egrep -vw ${build_packages[0]} $buildorder > $buildorder.2 + mv $buildorder.2 $buildorder + unset build_packages + popd > /dev/null +done + +pkgs=$(cat $build_dir/log | grep "built:") && { + error "Those packages were built and installed:" + plain "$(echo ${pkgs[@]} | cut -d: -f2)" + msg "Uploading packages to the server" + librerelease +} +pkgs=$(cat $build_dir/log | grep "failed:") && { + error "Those packages failed to build:" + plain "$(echo ${pkgs[@]} | cut -d: -f2)" +} +pkgs=$(cat $build_dir/log | grep "unstaged:") && { + error "Those packages couldn't be staged because of missing reponame:" + plain "$(cat $build_dir/log | grep "unstaged:" | cut -d: -f2)" +} + +exit 0 \ No newline at end of file diff --git a/libretools.conf b/libretools.conf index 8a7769f..70479a3 100644 --- a/libretools.conf +++ b/libretools.conf @@ -50,8 +50,13 @@ ABSLIBREGIT=http://projects.parabolagnulinux.org/abslibre.git/ #COMMITCMD=git #COMMITCMD=hg +## Build cmd for fullpkg-ng +## Uncomment one of those or make one of your choice +#FULLBUILDCMD="makepkg --noconfirm --nocheck -sciL" +#FULLBUILDCMD="here is a place for cross-compiling build cmd" + # Checks if vars aren't empty -# + for VAR in CHROOTDIR CHROOT CHCOPY CACHEDIR PARABOLAHOST LIBREDESTDIR \ LIBRESRCDIR BLACKLIST WORKDIR PATCHDIR REPOS ARCHES ABSLIBREGIT COMMITCMD DIFFTOOL; do -- cgit v1.2.3-2-g168b From c62355a305b1a232270e0cfc8c38e33b4339d3b5 Mon Sep 17 00:00:00 2001 From: Joshua Ismael Haase Hernandez Date: Wed, 25 May 2011 02:22:43 -0500 Subject: coexisting fullpkg, fullpkg-ng --- fullpkg | 385 ++++++++++++++++++++++++++++++---------------------------------- 1 file changed, 181 insertions(+), 204 deletions(-) diff --git a/fullpkg b/fullpkg index a428440..974dc85 100755 --- a/fullpkg +++ b/fullpkg @@ -1,16 +1,15 @@ #!/bin/bash # TODO +# * Do version checking +# * Detect circular builds # * Detect pkgnames by provides, replaces, etc. instead of dir tree -eval $(egrep "PACKAGER|CARCH" /etc/makepkg.conf) -eval $(egrep "ABSROOT" /etc/abs.conf) -eval $(egrep "REPOS|source" /etc/libretools.conf) -source /usr/bin/libremessages +source /etc/makepkg.conf +source /etc/abs.conf +source /etc/libretools.conf -[[ -r $XDG_CONFIG_HOME/libretools/libretools.conf ]] && \ - source $XDG_CONFIG_HOME/libretools/libretools.conf - -## START FUNCTIONS ## +[[ -r ~/.config/libretools/libretools.conf ]] && \ + source ~/.config/libretools/libretools.conf function usage { echo "cd to a dir containing a PKGBUILD and run:" @@ -20,13 +19,42 @@ function usage { echo echo "OPTIONS:" echo " -h : this message." - echo " -c : check deps" - # printf " -f pkgname : build even when a package has been built. " - # printf " Use it as many times as needed\n" - echo " -a absdir : set ABSROOT to this dir" + echo " -f : build even when a package has been built." + echo " -n absdir : set ABSROOT to this dir" + echo " -r reponame : set repo name to reponame" + echo " -R pkgname : build pkgname if it is a dep" + echo +} + +force_build='n' +force_array=() +_fullpkgargs="" +failed=() +missing=() + +while getopts 'hfn:r:R:' arg; do + case $arg in + h) usage; exit 0 ;; + f) force_build='y' ;; + R) force_array=(${force_array[@]} $OPTARG); _fullpkgargs+="-R $OPTARG ";; + n) ABSROOT="$OPTARG" ;; + r) repo="$OPTARG" ;; + esac +done + +[[ ! -r PKGBUILD ]] && { + error "This isn't a build directory" echo + usage + exit 1 } +tmp_dir=$(mktemp -d /tmp/$(basename $PWD).XXXXXX) +queue_file=$(mktemp /tmp/queue.XXXXXX) +ban_file=$(mktemp /tmp/ban.XXXXXX) + +## START FUNCTIONS ## + # Queue Management # * Always get the queue list from the server # * Add/Remove from queue @@ -37,7 +65,7 @@ function usage { # Get the queue list from the server get_queue() { - wget -N http://repo.parabolangunlinux.org/files/queue $queue_file >/dev/null 2>&1 || { + rsync -e ssh -aq $PARABOLAHOST:mips64el/queue $queue_file >/dev/null 2>&1 || { error "Failed to retrieve queue list" return 1 } @@ -54,7 +82,9 @@ put_queue() { # Add packages to the queue update_queue() { get_queue || return $? + basename $PWD | sed "s/$/:$PACKAGER/" >> $queue_file || return 2 + put_queue || return $? } @@ -65,14 +95,14 @@ remove_queue() { grep -vw "^$(basename $PWD)" $queue_file > $queue_file.2 cat $queue_file.2 > $queue_file - put_queue && rm $queue_file.2 && return 0 || return $? + put_queue && rm $queue_file{,.2} && return 0 || return $? } # Checks if a package is listed check_queue() { get_queue || return $? - local packager=$(grep -w "$(basename $PWD)" ${queue_file} | cut -d ':' -f2) + packager=$(grep -w "$(basename $PWD)" ${queue_file} | cut -d ':' -f2) [[ ! -z $packager ]] && [[ "$packager" != "$PACKAGER" ]] && { warning "$(basename $PWD) is being packaged by $packager. Please wait." @@ -84,214 +114,161 @@ check_queue() { # END Queue Management # +# Checks if the package is banned from building +is_banned() { + rsync -e ssh -aq $PARABOLAHOST:mips64el/ban $ban_file >/dev/null 2>&1 || { + plain "Failed to get ban list" + return 1 + } + grep -w $1 $ban_file >/dev/null 2>&1 + return $? +} + guess_repo() { basename $(dirname $(pwd)) } -# Usage: cleanup [ $(basename $PWD) ] from PKGBUILD dir -# cleans the tempdir -function cleanup { - if [ ! -d $tempdir/ ]; then - return 1 - elif [ -d $tempdir -a ${#@} -gt 0 ]; then - for _dir in $@; do - rm -rf $tempdir/$_dir/ - done - fi +# usage : in_array( $needle, $haystack ) +# return : 0 - found +# 1 - not found +function in_array { + [[ $2 ]] || return 1 + local needle=$1; shift + local item + for item in "$@"; do + [[ ${item#@} = $needle ]] && return 0 + done + return 1 # Not Found } -# Usage: find_deps -# no parameters -function find_deps { - [ $level -lt 20 ] || return 20 - ## Check this level. - source PKGBUILD - local repo=${repo:-$(guess_repo)} - - # TODO: If this package is in force_build: skip this step - # If package is built exit - if is_built "${pkgbase:-${pkgname[0]}}>=${pkgver}-${pkgrel}"; - then - msg2 "${pkgbase:-${pkgname[0]}}>=${pkgver}-${pkgrel} : built"; exit 0 - fi - - msg2 "${pkgbase:-${pkgname[0]}}>=${pkgver}-${pkgrel}" - - # Tell which packages are deeper in deps (even if they are on tempdir) - # so we can build them first. - echo "${level}: $(basename $PWD)" >> "${tempdir}/.BUILDORDER" - # if pkgbuild directory is on tempdir, do not copy and exit - if [ -d "${tempdir}/$(basename $PWD)" ]; then - exit 0 - else - cp -r ../$(basename $PWD) ${tempdir}/ - # Info to eval later - echo "repo=$repo" > "${tempdir}/$(basename $PWD)/.INFO" - fi +function quit { + remove_queue + exit 1 +} - ## Check next levels - # Clean version checking - deps=$(echo "${depends[@]} ${makedepends[@]} ${pkgdeps[@]}" | \ - sed "s/[=<>]\+[^ ]\+//g" | \ - tr ' ' "\n" | \ - sort -u) - declare -i next_level=$level+1 - for _dep in ${deps[@]}; do - for _repo in ${REPOS[@]}; do - # try to find $_dep on each repo from dirname - [ -e "$ABSROOT/${_repo}/$_dep/PKGBUILD" ] && { - pushd "$ABSROOT/${_repo}/$_dep" >/dev/null - "$0 -c '${check_deps_file}' -d '${tempdir}' -l '${next_level}'" - # Circular deps must fail - [ $? -eq 20 ] && return 20 - popd >/dev/null - # found, go to next dep - break 1 - } - # search pkgname in repo if that doesn't work - # this should find pkgsplits - _dir=$(find "$ABSROOT/${_repo}" -type f -name PKGBUILD -print0 | \ - "xargs" -0 -e grep -H -E "pkgname|pkgbase" | grep $_dep) && { - pushd $(dirname $(echo $_dir | cut -d: -f1)) >/dev/null - "$0 -c '${check_deps_file}' -d '${tempdir}' -l '${next_level}'" - # Circular deps must fail - [ $? -eq 20 ] && return 20 - popd > /dev/null - # found, go to next dep - break 1 - } - done - done - unset next_level dir +function cleanup { + rm $ban_file $queue_file + rm -rf $tmp_dir } -## END FUNCTIONS ## +# TODO keep track of spawned fullpkgs -force_build="" -level=0 -while getopts 'ha:c:d:l:' arg; do - case $arg in - h) usage; exit 0 ;; - a) ABSROOT="$OPTARG" ;; - c) check_deps_file="$OPTARG" ;; - # f) force_build+="-f pkgname " ;; - d) tempdir="$OPTARG" ;; - # hidden function to know what to build first. - # if $level > 0 it will not build - l) level=$OPTARG ;; - esac -done -mkdir -p $XDG_CONFIG_HOME/libretools -queue_file=$XDG_CONFIG_HOME/libretools/queue -ban_file=$XDG_CONFIG_HOME/libretools/ban -touch $queue_file $ban_file -# Only on level 0 -[ $level -eq 0 ] && { - # if tempdir exist use it, else make a tempdir - tempdir="$(mktemp -d /tmp/fullpkg.XXXXXX)" - check_deps_file=$tempdir/deps - msg "Updating pacman db and packages" - sudo pacman -Syu --noconfirm - msg "Checking dependencies" -} -buildorder=$tempdir/.BUILDORDER -touch $buildorder $tempdir/deps $tempdir/false $tempdir/install -touch $check_deps_file +## END FUNCTIONS ## -[[ ! -r PKGBUILD ]] && { - error "This isn't a build directory" - usage && exit 1 -} +source PKGBUILD +repo=${repo:-$(guess_repo)} +msg "Building ${repo:-missing repo}/${pkgbase:-${pkgname[@]}}: $pkgdesc" -## if $level = 20 there is highly likely there are circular deps -[ $level -eq 20 ] && exit 20 - -find_deps || { - # if find_deps finds circular deps - # it should exit with status 20 - [ $? -eq 20 ] && \ - # only show message on level 0 - [ $level -eq 0 ] && \ - error "Check for circular deps on $tempdir/.BUILDORDER"; - exit 20 -} +# Pre build tests +if [ $force_build == 'n' ]; then -# levels greater than 0 must only check deps -[ $level -gt 0 ] && exit 0 - - -# check .BUILDORDER to not include banned deps and -[ $level -eq 0 -a -d $tempdir ] && { - # Check for banned deps - if [ -w $ban_file -a -r $ban_file ]; then - chmod o+rw $ban_file || error "Ban file is not readable/writable ($ban_file)" - else - wget -N http://repo.parabolagnulinux.org/files/ban $ban_file >/dev/null 2>&1 || \ - warning "Failed to get ban list" && [ -r $ban_file ] && { - # continue if download failed but local copy - search=$(cat $ban_file | tr "\n" "|") - echo ${@} | tr " " "\n" | egrep -w $search $buildorder | cleanup - echo ${@} | tr " " "\n" | egrep -vw $search $buildorder > $buildorder.2 - mv $buildorder.2 $buildorder - unset search - } + # Be able to write files + if [[ ! -w $queue_file ]]; then + error "can't write queue file" + exit 1 + elif [[ ! -w $ban_file ]] ; then + error "can't write ban file" + exit 1 fi -} -## START Building -msg "Building packages:" -pushd "$tempdir" - -[ ! -w $queue_file ] && error "can't write queue file" - -trap "break" INT -trap "remove_queue" EXIT INT QUIT TERM KILL -build_packages=$(sort -gr $buildorder | cut -d: -f2) -while [ ${#build_packages[@]} -gt 0 ]; do - build_packages=$(sort -gr $buildorder | cut -d: -f2) - pushd $temp_dir/${build_packages[0]} - egrep -vw ${build_packages[0]} $buildorder > $buildorder.2 - mv $buildorder.2 $buildorder - unset build_packages - eval $(egrep "pkgname|pkgbase|pkgver|pkgrel" PKGBUILD) - msg2 "${pkgbase:-${pkgname[0]}} $pkgver-$pkgrel" - if ! grep mips64el PKGBUILD >/dev/null; then - plain "Adding mips64el arch" - sed -i "s/^\(arch=([^)anym]\+\))/\1 'mips64el')/" "PKGBUILD" + if is_built "${pkgbase:-${pkgname[0]}}>=${pkgver}-${pkgrel}"; then + msg2 "This package is built." + exit 0 fi - # Let everybody know we're building this. - update_queue || { - warning "Couldn't update the queue, let your partners know about this." - } - makepkg --noconfirm --nocheck -smciL; r=$? - case $r in - 0) plain "The build was succesful." - source .INFO && [ -n $repo ] && librestage $repo || \ - echo "unstaged: $(basename $PWD)" >> $tempdir/failed - echo "$(basename $PWD)" >> $tempdir/install - cleanup "$(basename $PWD)" - msg2 "Sync db again" - sudo pacman -Sy ;; - 1) error "There were errors while trying to build the package." - echo "failed: $(basename $PWD)" >> $tempdir/failed ;; - 2) error "The build failed." && echo "failed: $(basename $PWD)" >> $tempdir/failed ;; - esac - popd >/dev/null -done -msg "Those packages were installed" -plain $(cat $tempdir/install | tr "\n" " ") + if is_banned ${pkgbase:-$pkgname}; then + error "This package is banned from building. Check the ban list" + exit 1 + fi -msg "Uploading packages to the server" -librerelease + check_queue || exit 1 + +fi + +# This will be executed at exit for any reason. +trap "quit" EXIT INT QUIT TERM KILL HUP + +if ! grep mips64el PKGBUILD >/dev/null; then + msg "Adding mips64el arch" + sed -i "s/^\(arch=([^)anym]\+\))/\1 'mips64el')/" "PKGBUILD" +fi + +# Clean version checking +deps=$(echo "${depends[@]} ${makedepends[@]} ${pkgdeps[@]}" | \ + sed "s/[=<>]\+[^ ]\+//g" | \ + tr ' ' "\n" | \ + sort -u) + +msg "Checking dependencies" +for _dep in ${deps[@]}; do + is_banned $_dep && continue + + for _repo in ${REPOS[@]}; do + # TODO find split packages + [[ -e "$ABSROOT/${_repo}/$_dep/PKGBUILD" ]] && { + source "$ABSROOT/${_repo}/$_dep/PKGBUILD" + msg2 "Checking for $_dep>=$pkgver-$pkgrel" + + if ! in_array $_dep ${force_array[@]}; then + if is_built "$_dep>=$pkgver-$pkgrel"; then + plain "this package is built" + break + fi + else + _fullpkgargs+="-f " + _fullpkgargs="$(echo $_fullpkgargs | sed s/"-R $_dep "//)" + force_array=( $(echo ${forcearray[@]} | tr " " "\n" | grep -vw "^$_dep") ) + fi + + cp -r "$ABSROOT/$_repo/$_dep" $tmp_dir/ || { + error "Can't copy $_dep to the work dir." + exit 1 + } + + # Enter the work dir and run this command in it + pushd $tmp_dir/$_dep >/dev/null + + $0 -r $_repo $_fullpkgargs + + [[ $? -ne 0 ]] && { + failed=(${failed[@]} $_dep) + } + + popd >/dev/null + } + done +done -pkgs=$(cat $tempdir/failed | grep "failed:") && { - error "Those packages failed to build:" - plain "$(echo ${pkgs[@]} | cut -d: -f2)" +# TODO probably not elegant enough +# TODO only the last fullpkg should show this message +# and it should contain all failed pkgs +[[ ${#failed[@]} -gt 0 ]] && { + error "This packages failed to build: ${failed[@]}" + exit 1 } -pkgs=$(cat $tempdir/failed | grep "unstaged:") && { - error "Those packages couldn't be staged because of missing reponame:" - plain "$(cat $tempdir/failed | grep "unstaged:" | cut -d: -f2)" + +# Let everybody know we're building this +update_queue || { + warning "Couldn't update the queue, let your partners know about this." } -exit 0 \ No newline at end of file +cp -r ../$(basename $PWD) $tmp_dir/ +pushd $tmp_dir/$(basename $PWD) >/dev/null + +msg "Syncing database" +sudo pacman -Syu --noconfirm +makepkg --noconfirm --nocheck -sLcr ; r=$? +case $r in + 0) msg "The build was succesful." + mipsrelease *.pkg.tar.* + librestage $repo + librerelease + sudo pacman -Sy + # cleanup is only on succesfull build so failed can be inspected + cleanup;; + 1) error "There were errors while trying to build the package." ;; + 2) error "The build failed." ;; +esac + +exit $r + -- cgit v1.2.3-2-g168b