diff options
| -rwxr-xr-x | any-to-ours | 71 | ||||
| -rw-r--r-- | config | 38 | ||||
| -rwxr-xr-x | db-check-nonfree | 46 | ||||
| -rwxr-xr-x | db-sync | 212 | ||||
| -rw-r--r-- | db-sync.conf | 7 |
5 files changed, 15 insertions, 359 deletions
diff --git a/any-to-ours b/any-to-ours deleted file mode 100755 index 8a4e874..0000000 --- a/any-to-ours +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash -# Releases 'any' packages from Arch arches to ours - -trap_exit() { - echo - error "$@" - exit 1 -} - -source "$(dirname "$(readlink -e "$0")")/config" -source "$(librelib messages)" - -# From makepkg -set -E - -trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT -trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT -trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR - -# The architecture to compare with -BASEARCH='x86_64' - -# Traverse all Arch repos -for _repo in "${ARCHREPOS[@]}"; do - msg "Processing %s..." "${_repo}" - - # Find 'any' packages - # This is hardcoded but it could release other arches... - PKGS=($(find "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \ - -iname '*-any.pkg.tar.?z' \ - -printf "%f ")) - - if [ ${#PKGS[@]} -eq 0 ]; then - msg2 "No '%s' packages here" any - continue - fi - - for _arch in "${OURARCHES[@]}"; do - msg2 "Syncing %s..." "${_arch}" - - # Sync 'any' only and extract the synced packages - SYNCED=($( - rsync -av \ - --include='*-any.pkg.tar.?z' \ - --include='*-any.pkg.tar.?z.sig' \ - --exclude='*' \ - "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \ - "${FTP_BASE}/${_repo}/os/${_arch}/" 2>&1 | \ - grep 'any\.pkg\.tar\..z$' | \ - cut -d ' ' -f 1 )) - - if [ ${#SYNCED[@]} -eq 0 ]; then - msg2 "Already synced (or error happened)" - continue - fi - - msg2 "Synced %d packages: %s" "${#SYNCED[@]}" "${SYNCED[*]}" - - msg2 "Adding to db..." - - pushd "${FTP_BASE}/${_repo}/os/${_arch}/" >/dev/null - - # Add the packages to the db - repo-add "${_repo}${DBEXT}" "${SYNCED[@]}" - - popd >/dev/null - - # Avoid mixups - unset SYNCED PKGS - done -done @@ -2,29 +2,26 @@ FTP_BASE="/srv/http/repo/public" SVNREPO="/var/abs" -# Repos from Arch -ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib' 'multilib-testing') TESTING_REPO='testing' -# Official Parabola repos -OURREPOS=('libre' 'libre-testing' 'libre-multilib' 'libre-multilib-testing') -# User repos -USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan') -# Community project repos -PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'nonprism') -# Remote repos -PKGREPOS=("${ARCHREPOS[@]}" "${OURREPOS[@]}" "${USERREPOS[@]}" "${PROJREPOS[@]}") +PKGREPOS=( + # Arch + {core,extra,testing,staging} + {gnome,kde}-unstable + {community,multilib}{,-testing,-staging} + # Parabola + libre{,-testing} + libre-multilib{,-testing} + pcr kernels cross java nonprism + '~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan') +) PKGPOOL='pool/parabola' SRCPOOL='sources/parabola' # Directories where packages are shared between repos # *relative to FTP_BASE* -ARCHPKGPOOLS=(pool/{packages,community}) -OURPKGPOOLS=(pool/parabola) -PKGPOOLS=(${OURPKGPOOLS[@]} ${ARCHPKGPOOLS[@]}) +PKGPOOLS=(pool/{packages,community,parabola}) # Directories where sources are stored -ARCHSRCPOOLS=(sources/{packages,community}) -OURPKGPOOLS=(sources/parabola) -SRCPOOLS=(${OURSRCPOOLS[@]} ${ARCHSRCPOOLS[@]}) +SRCPOOLS=(sources/{packages,community,parabola}) CLEANUP_DESTDIR="$FTP_BASE/old/packages" CLEANUP_DRYRUN=false @@ -43,16 +40,11 @@ LOCK_TIMEOUT=300 [ -n "${STAGING:-}" ] || STAGING="$HOME/staging/unknown/staging" TMPDIR="/tmp" -ARCHARCHES=(i686 x86_64) -OURARCHES=(mips64el) -ARCHES=(${ARCHARCHES[@]} ${OURARCHES[@]}) +ARCHES=(i686 x86_64 mips64el) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.xz" SRCEXT=".src.tar.gz" +# Used by cron-jobs/sourceballs2 MAKEPKGCONF="~/.makepkg.conf" -BLACKLIST_FILE="$HOME/blacklist/blacklist.txt" - -# parabolaweb root -WEB_DIR=/srv/http/parabolagnulinux.org/web diff --git a/db-check-nonfree b/db-check-nonfree deleted file mode 100755 index 37b7cf6..0000000 --- a/db-check-nonfree +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -. "$(dirname "$(readlink -e "$0")")/config" -. "$(dirname "$(readlink -e "$0")")/db-functions" - -if [ $# -ge 1 ]; then - error "Calling %s with a specific repository is not supported" "${0##*/}" - exit 1 -fi - -# TODO: this might lock too much (architectures) -for repo in "${repos[@]}"; do - for pkgarch in "${ARCHES[@]}"; do - repo_lock "${repo}" "${pkgarch}" || exit 1 - done -done - -msg "Check nonfree in repo:" -nonfree=($(cut -d: -f1 "${BLACKLIST_FILE}" | sort -u)) -for repo in "${ARCHREPOS[@]}"; do - for pkgarch in "${ARCHES[@]}"; do - msg2 "%s %s" "$repo" "$pkgarch" - if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then - continue - fi - unset dbpkgs - unset cleanpkgs - cleanpkgs=() - dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) - for pkgname in "${dbpkgs[@]}"; do - if in_array "${pkgname}" "${nonfree[@]}"; then - cleanpkgs+=("${pkgname}") - fi - done - if [ ${#cleanpkgs[@]} -ge 1 ]; then - msg2 "Nonfree: %s" "${cleanpkgs[*]}" - arch_repo_remove "${repo}" "${pkgarch}" "${cleanpkgs[@]}" - fi - done -done - -for repo in "${repos[@]}"; do - for pkgarch in "${ARCHES[@]}"; do - repo_unlock "${repo}" "${pkgarch}" - done -done diff --git a/db-sync b/db-sync deleted file mode 100755 index b26e917..0000000 --- a/db-sync +++ /dev/null @@ -1,212 +0,0 @@ -#!/bin/bash -# Syncs Arch repos based on info contained in repo.db files -# License: GPLv3 - -# Principles -# * Get repo.db from an Arch-like repo -# * Generate a list of available packages -# * Create sync whitelist (based on package blacklist) -# * Get packages -# * Check package signatures -# * Check database signatures -# * Sync repo => repo - -# TODO -# * make a tarball of files used for forensics - -# Run as `V=true db-sync` to get verbose output -VERBOSE=${V} -extra=() -${VERBOSE} && extra+=(-v) - -WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") -trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT - -# Returns contents of a repo -get_repos() { - # Exclude everything but db files - rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \ - --include="*.db" \ - --include="*${DBEXT}" \ - --include="*.files" \ - --include="*${FILESEXT}" \ - --exclude="*" \ - --delete-after \ - "rsync://${mirror}/${mirrorpath}/" "$WORKDIR" -} - -get_repo_content() { - # Return all contents - bsdtar tf "${1}" | \ - cut -d "/" -f 1 | \ - sort -u -} - -# Prints blacklisted packages -get_blacklist() { - cut -d ':' -f 1 "${BLACKLIST_FILE}" -} - -# repo -# arch -get_repo_file() { - echo "${WORKDIR}/${1}/os/${2}/${1}" -} - -# Process the databases and get the libre packages -init() { - - # Get the blacklisted packages - blacklist=($(get_blacklist)) - # Store all the whitelist files - whitelists=() - - msg "%d packages in blacklist" ${#blacklist[@]} - - test ${#blacklist[@]} -eq 0 && fatal_error "Empty blacklist" - - # Sync the repos databases - get_repos - - # Traverse all repo-arch pairs - for _repo in "${ARCHREPOS[@]}"; do - for _arch in "${ARCHARCHES[@]}"; do - msg "Processing %s-%s" "${_repo}-${_arch}" - - db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT} - files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT} - - if [ ! -f "${db_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${db_file}" - continue - fi - if [ ! -f "${files_file}" ]; then - warning "%s doesn't exist, skipping this repo-arch" "${files_file}" - continue - fi - - # Remove blacklisted packages and count them - # TODO capture all removed packages for printing on debug mode - msg2 "Removing blacklisted packages from %s database..." .db - LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \ - |& sed -n 's/-> Removing/ &/p' - msg2 "Removing blacklisted packages from %s database..." .files - LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \ - |& sed -n 's/-> Removing/ &/p' - # Get db contents - db=($(get_repo_content "${db_file}")) - - msg2 "Process clean db for syncing..." - - # Create a whitelist, add * wildcard to end - # TODO due to lack of -arch suffix, the pool sync retrieves every arch even if - # we aren't syncing them - # IMPORTANT: the . in the sed command is needed because an empty - # whitelist would consist of a single * allowing any package to - # pass through - printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist" - - msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)" - - # Sync excluding everything but whitelist - # We delete here for cleanup - rsync "${extra[@]}" --no-motd -rtlH \ - --delete-after \ - --delete-excluded \ - --delay-updates \ - --include-from="/tmp/${_repo}-${_arch}.whitelist" \ - --exclude="*" \ - "rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/" \ - "${FTP_BASE}/${_repo}/os/${_arch}/" - - # Add a new whitelist - whitelists+=(/tmp/${_repo}-${_arch}.whitelist) - - msg "Putting databases back in place" - rsync "${extra[@]}" --no-motd -rtlH \ - --delay-updates \ - --safe-links \ - "${WORKDIR}/${_repo}/os/${_arch}/" \ - "${FTP_BASE}/${_repo}/os/${_arch}/" - - # Cleanup - unset db - done - done - - - msg "Syncing package pool" - # Concatenate all whitelists, check for single *s just in case - cat "${whitelists[@]}" | grep -v "^\*$" | sort -u > /tmp/any.whitelist - - msg2 "Retrieving %d packages from pool" "$(wc -l /tmp/any.whitelist | cut -d' ' -f1)" - - # Sync - # *Don't delete-after*, this is the job of cleanup scripts. It will remove our - # packages too - local pkgpool - for pkgpool in "${ARCHPKGPOOLS[@]}"; do - rsync "${extra[@]}" --no-motd -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - "rsync://${mirror}/${mirrorpath}/${pkgpool}/" \ - "${FTP_BASE}/${pkgpool}/" - done - - # Sync sources - msg "Syncing source pool" - #sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist - #msg2 "Retrieving %d sources from pool" $(wc -l < /tmp/any-src.whitelist) - - # Sync - # *Don't delete-after*, this is the job of cleanup scripts. It will remove our - # packages too - local srcpool - for srcpool in "${ARCHSRCPOOLS[@]}"; do - rsync "${extra[@]}" --no-motd -rtlH \ - --delay-updates \ - --safe-links \ - --include-from=/tmp/any.whitelist \ - --exclude="*" \ - "rsync://${mirror}/${mirrorpath}/${srcpool}/" \ - "${FTP_BASE}/${srcpool}/" - done - - # Cleanup - unset blacklist whitelists _arch _repo repo_file -} - -fatal_error() { - local mesg=$1; shift - error "$mesg" "$@" - exit 1 -} - -trap_exit() { - local signal=$1; shift - echo - error "$@" - trap -- "$signal" - kill "-$signal" "$$" -} - -source "$(dirname "$(readlink -e "$0")")/config" -source "$(dirname "$(readlink -e "$0")")/db-sync.conf" -source "$(librelib messages)" - -# Check variables presence -for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do - test -z "${!var}" && fatal_error "Empty %s" "${var}" -done - -# From makepkg -set -E -for signal in TERM HUP QUIT; do - trap "trap_exit $signal '%s signal caught. Exiting...' $signal" "$signal" -done -trap 'trap_exit INT "Aborted by user! Exiting..."' INT -trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR - -init diff --git a/db-sync.conf b/db-sync.conf deleted file mode 100644 index 3d6c709..0000000 --- a/db-sync.conf +++ /dev/null @@ -1,7 +0,0 @@ -#mirror="mirrors.uk2.net" -mirror="mirrors.kernel.org" -#mirror="mirror.umd.edu" -#mirror="archlinux.c3sl.ufpr.br" -#mirror="mirror.us.leaseweb.net" -#mirror="mirror.de.leaseweb.net" -mirrorpath="archlinux" |