Remove db-sync and friends. Simplify the config accordingly

5 files changed, 15 insertions, 359 deletions

diff --git a/any-to-ours b/any-to-ours
deleted file mode 100755
index 8a4e874..0000000
--- a/any-to-ours
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/bin/bash
-# Releases 'any' packages from Arch arches to ours
-
-trap_exit() {
-	echo
-	error "$@"
-	exit 1
-}
-
-source "$(dirname "$(readlink -e "$0")")/config"
-source "$(librelib messages)"
-
-# From makepkg
-set -E
-
-trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT
-trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT
-trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR
-
-# The architecture to compare with
-BASEARCH='x86_64'
-
-# Traverse all Arch repos
-for _repo in "${ARCHREPOS[@]}"; do
-	msg "Processing %s..." "${_repo}"
-
-	# Find 'any' packages
-	# This is hardcoded but it could release other arches...
-	PKGS=($(find "${FTP_BASE}/${_repo}/os/${BASEARCH}/" \
-	             -iname '*-any.pkg.tar.?z' \
-	             -printf "%f "))
-
-	if [ ${#PKGS[@]} -eq 0 ]; then
-		msg2 "No '%s' packages here" any
-		continue
-	fi
-
-	for _arch in "${OURARCHES[@]}"; do
-		msg2 "Syncing %s..." "${_arch}"
-
-		# Sync 'any' only and extract the synced packages
-		SYNCED=($(
-				rsync -av \
-					--include='*-any.pkg.tar.?z' \
-					--include='*-any.pkg.tar.?z.sig' \
-					--exclude='*' \
-					"${FTP_BASE}/${_repo}/os/${BASEARCH}/" \
-					"${FTP_BASE}/${_repo}/os/${_arch}/" 2>&1 | \
-					grep 'any\.pkg\.tar\..z$' | \
-					cut -d ' ' -f 1 ))
-
-		if [ ${#SYNCED[@]} -eq 0 ]; then
-			msg2 "Already synced (or error happened)"
-			continue
-		fi
-
-		msg2 "Synced %d packages: %s" "${#SYNCED[@]}" "${SYNCED[*]}"
-
-		msg2 "Adding to db..."
-
-		pushd "${FTP_BASE}/${_repo}/os/${_arch}/" >/dev/null
-
-		# Add the packages to the db
-		repo-add "${_repo}${DBEXT}" "${SYNCED[@]}"
-
-		popd >/dev/null 
-
-		# Avoid mixups
-		unset SYNCED PKGS
-	done
-done
diff --git a/config b/config
index e43c13d..87fbf2d 100644
--- a/config
+++ b/config
@@ -2,29 +2,26 @@
 FTP_BASE="/srv/http/repo/public"
 SVNREPO="/var/abs"
 
-# Repos from Arch
-ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib' 'multilib-testing')
 TESTING_REPO='testing'
-# Official Parabola repos
-OURREPOS=('libre' 'libre-testing' 'libre-multilib' 'libre-multilib-testing')
-# User repos
-USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan')
-# Community project repos
-PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'nonprism')
-# Remote repos
-PKGREPOS=("${ARCHREPOS[@]}" "${OURREPOS[@]}" "${USERREPOS[@]}" "${PROJREPOS[@]}")
+PKGREPOS=(
+	# Arch
+	{core,extra,testing,staging}
+	{gnome,kde}-unstable
+	{community,multilib}{,-testing,-staging}
+	# Parabola
+	libre{,-testing}
+	libre-multilib{,-testing}
+	pcr kernels cross java nonprism
+	'~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan')
+)
 PKGPOOL='pool/parabola'
 SRCPOOL='sources/parabola'
 
 # Directories where packages are shared between repos
 # *relative to FTP_BASE*
-ARCHPKGPOOLS=(pool/{packages,community})
-OURPKGPOOLS=(pool/parabola)
-PKGPOOLS=(${OURPKGPOOLS[@]} ${ARCHPKGPOOLS[@]})
+PKGPOOLS=(pool/{packages,community,parabola})
 # Directories where sources are stored
-ARCHSRCPOOLS=(sources/{packages,community})
-OURPKGPOOLS=(sources/parabola)
-SRCPOOLS=(${OURSRCPOOLS[@]} ${ARCHSRCPOOLS[@]})
+SRCPOOLS=(sources/{packages,community,parabola})
 
 CLEANUP_DESTDIR="$FTP_BASE/old/packages"
 CLEANUP_DRYRUN=false
@@ -43,16 +40,11 @@ LOCK_TIMEOUT=300
 
 [ -n "${STAGING:-}" ] || STAGING="$HOME/staging/unknown/staging"
 TMPDIR="/tmp"
-ARCHARCHES=(i686 x86_64)
-OURARCHES=(mips64el)
-ARCHES=(${ARCHARCHES[@]} ${OURARCHES[@]})
+ARCHES=(i686 x86_64 mips64el)
 DBEXT=".db.tar.gz"
 FILESEXT=".files.tar.gz"
 PKGEXT=".pkg.tar.xz"
 SRCEXT=".src.tar.gz"
 
+# Used by cron-jobs/sourceballs2
 MAKEPKGCONF="~/.makepkg.conf"
-BLACKLIST_FILE="$HOME/blacklist/blacklist.txt"
-
-# parabolaweb root
-WEB_DIR=/srv/http/parabolagnulinux.org/web
diff --git a/db-check-nonfree b/db-check-nonfree
deleted file mode 100755
index 37b7cf6..0000000
--- a/db-check-nonfree
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/bash
-
-. "$(dirname "$(readlink -e "$0")")/config"
-. "$(dirname "$(readlink -e "$0")")/db-functions"
-
-if [ $# -ge 1 ]; then
-	error "Calling %s with a specific repository is not supported" "${0##*/}"
-	exit 1
-fi
-
-# TODO: this might lock too much (architectures)
-for repo in "${repos[@]}"; do
-	for pkgarch in "${ARCHES[@]}"; do
-		repo_lock "${repo}" "${pkgarch}" || exit 1
-	done
-done
-
-msg "Check nonfree in repo:"
-nonfree=($(cut -d: -f1 "${BLACKLIST_FILE}" | sort -u))
-for repo in "${ARCHREPOS[@]}"; do
-	for pkgarch in "${ARCHES[@]}"; do
-		msg2 "%s %s" "$repo" "$pkgarch"
-		if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then
-			continue
-		fi
-		unset dbpkgs
-		unset cleanpkgs
-		cleanpkgs=()
-		dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u ))
-		for pkgname in "${dbpkgs[@]}"; do
-			if in_array "${pkgname}" "${nonfree[@]}"; then
-				cleanpkgs+=("${pkgname}")
-			fi
-		done
-		if [ ${#cleanpkgs[@]} -ge 1 ]; then
-			msg2 "Nonfree: %s" "${cleanpkgs[*]}"
-			arch_repo_remove "${repo}" "${pkgarch}" "${cleanpkgs[@]}"
-		fi
-	done
-done
-
-for repo in "${repos[@]}"; do
-	for pkgarch in "${ARCHES[@]}"; do
-		repo_unlock "${repo}" "${pkgarch}"
-	done
-done
diff --git a/db-sync b/db-sync
deleted file mode 100755
index b26e917..0000000
--- a/db-sync
+++ /dev/null
@@ -1,212 +0,0 @@
-#!/bin/bash
-# Syncs Arch repos based on info contained in repo.db files
-# License: GPLv3
-
-# Principles
-# * Get repo.db from an Arch-like repo
-# * Generate a list of available packages
-# * Create sync whitelist (based on package blacklist)
-# * Get packages
-# * Check package signatures
-# * Check database signatures
-# * Sync repo => repo
-
-# TODO
-# * make a tarball of files used for forensics
-
-# Run as `V=true db-sync` to get verbose output
-VERBOSE=${V}
-extra=()
-${VERBOSE} && extra+=(-v)
-
-WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX")
-trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT
-
-# Returns contents of a repo
-get_repos() {
-	# Exclude everything but db files
-	rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \
-		--include="*.db" \
-		--include="*${DBEXT}" \
-		--include="*.files" \
-		--include="*${FILESEXT}" \
-		--exclude="*" \
-		--delete-after \
-		"rsync://${mirror}/${mirrorpath}/" "$WORKDIR"
-}
-
-get_repo_content() {
-	# Return all contents
-	bsdtar tf "${1}" | \
-		cut -d "/" -f 1 | \
-		sort -u
-}
-
-# Prints blacklisted packages
-get_blacklist() {
-	cut -d ':' -f 1 "${BLACKLIST_FILE}"
-}
-
-# repo
-# arch
-get_repo_file() {
-	echo "${WORKDIR}/${1}/os/${2}/${1}"
-}
-
-# Process the databases and get the libre packages
-init() {
-
-	# Get the blacklisted packages
-	blacklist=($(get_blacklist))
-	# Store all the whitelist files
-	whitelists=()
-
-	msg "%d packages in blacklist" ${#blacklist[@]}
-
-	test ${#blacklist[@]} -eq 0 && fatal_error "Empty blacklist"
-
-	# Sync the repos databases
-	get_repos
-
-	# Traverse all repo-arch pairs
-	for _repo in "${ARCHREPOS[@]}"; do
-		for _arch in "${ARCHARCHES[@]}"; do
-			msg "Processing %s-%s" "${_repo}-${_arch}"
-
-			db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT}
-			files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT}
-
-			if [ ! -f "${db_file}" ]; then
-				warning "%s doesn't exist, skipping this repo-arch" "${db_file}"
-				continue
-			fi
-			if [ ! -f "${files_file}" ]; then
-				warning "%s doesn't exist, skipping this repo-arch" "${files_file}"
-				continue
-			fi
-
-			# Remove blacklisted packages and count them
-			# TODO capture all removed packages for printing on debug mode
-			msg2 "Removing blacklisted packages from %s database..." .db
-			LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \
-			|& sed -n 's/-> Removing/	&/p'
-			msg2 "Removing blacklisted packages from %s database..." .files
-			LC_ALL=C repo-remove "${files_file}" "${blacklist[@]}" \
-			|& sed -n 's/-> Removing/	&/p'
-			# Get db contents
-			db=($(get_repo_content "${db_file}"))
-
-			msg2 "Process clean db for syncing..."
-
-			# Create a whitelist, add * wildcard to end
-			# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if
-			# we aren't syncing them
-			# IMPORTANT: the . in the sed command is needed because an empty
-			# whitelist would consist of a single * allowing any package to
-			# pass through
-			printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist"
-
-			msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)"
-
-			# Sync excluding everything but whitelist
-			# We delete here for cleanup
-			rsync "${extra[@]}" --no-motd -rtlH \
-				--delete-after \
-				--delete-excluded \
-				--delay-updates \
-				--include-from="/tmp/${_repo}-${_arch}.whitelist" \
-				--exclude="*" \
-				"rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/" \
-				"${FTP_BASE}/${_repo}/os/${_arch}/"
-
-			# Add a new whitelist
-			whitelists+=(/tmp/${_repo}-${_arch}.whitelist)
-
-			msg "Putting databases back in place"
-			rsync "${extra[@]}" --no-motd -rtlH \
-				--delay-updates \
-				--safe-links \
-				"${WORKDIR}/${_repo}/os/${_arch}/" \
-				"${FTP_BASE}/${_repo}/os/${_arch}/"
-
-			# Cleanup
-			unset db 
-		done
-	done
-
-
-	msg "Syncing package pool"
-	# Concatenate all whitelists, check for single *s just in case
-	cat "${whitelists[@]}" | grep -v "^\*$" | sort -u > /tmp/any.whitelist
-
-	msg2 "Retrieving %d packages from pool" "$(wc -l /tmp/any.whitelist | cut -d' ' -f1)"
-
-	# Sync
-	# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
-	# packages too
-	local pkgpool
-	for pkgpool in "${ARCHPKGPOOLS[@]}"; do
-		rsync "${extra[@]}" --no-motd -rtlH \
-			--delay-updates \
-			--safe-links \
-			--include-from=/tmp/any.whitelist \
-			--exclude="*" \
-			"rsync://${mirror}/${mirrorpath}/${pkgpool}/" \
-			"${FTP_BASE}/${pkgpool}/"
-	done
-
-	# Sync sources
-	msg "Syncing source pool"
-	#sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist
-	#msg2 "Retrieving %d sources from pool" $(wc -l < /tmp/any-src.whitelist)
-
-	# Sync
-	# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
-	# packages too
-	local srcpool
-	for srcpool in "${ARCHSRCPOOLS[@]}"; do
-		rsync "${extra[@]}" --no-motd -rtlH \
-			--delay-updates \
-			--safe-links \
-			--include-from=/tmp/any.whitelist \
-			--exclude="*" \
-			"rsync://${mirror}/${mirrorpath}/${srcpool}/" \
-			"${FTP_BASE}/${srcpool}/"
-	done
-
-	# Cleanup
-	unset blacklist whitelists _arch _repo repo_file
-}
-
-fatal_error() {
-	local mesg=$1; shift
-	error "$mesg" "$@"
-	exit 1
-}
-
-trap_exit() {
-	local signal=$1; shift
-	echo
-	error "$@"
-	trap -- "$signal"
-	kill "-$signal" "$$"
-}
-
-source "$(dirname "$(readlink -e "$0")")/config"
-source "$(dirname "$(readlink -e "$0")")/db-sync.conf"
-source "$(librelib messages)"
-
-# Check variables presence
-for var in DBEXT FILESEXT mirror mirrorpath WORKDIR BLACKLIST_FILE FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do
-	test -z "${!var}" && fatal_error "Empty %s" "${var}"
-done
-
-# From makepkg
-set -E
-for signal in TERM HUP QUIT; do
-	trap "trap_exit $signal '%s signal caught. Exiting...' $signal" "$signal"
-done
-trap 'trap_exit INT "Aborted by user! Exiting..."' INT
-trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR
-
-init
diff --git a/db-sync.conf b/db-sync.conf
deleted file mode 100644
index 3d6c709..0000000
--- a/db-sync.conf
+++ /dev/null
@@ -1,7 +0,0 @@
-#mirror="mirrors.uk2.net"
-mirror="mirrors.kernel.org"
-#mirror="mirror.umd.edu"
-#mirror="archlinux.c3sl.ufpr.br"
-#mirror="mirror.us.leaseweb.net"
-#mirror="mirror.de.leaseweb.net"
-mirrorpath="archlinux"