From c1bbe81b1e14f06ed49d9e0292ec9cd60903b104 Mon Sep 17 00:00:00 2001
From: Luke Shumaker <LukeShu@sbcglobal.net>
Date: Tue, 31 Dec 2013 14:11:53 -0500
Subject: add db-import

---
 db-import | 293 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 293 insertions(+)
 create mode 100755 db-import

(limited to 'db-import')

diff --git a/db-import b/db-import
new file mode 100755
index 0000000..7bc822f
--- /dev/null
+++ b/db-import
@@ -0,0 +1,293 @@
+#!/bin/bash -euE
+# Imports Arch-like repos, running them through a blacklist
+# License: GPLv3
+
+. $(dirname $0)/config
+. $(dirname $0)/db-import.conf
+. $(librelib messages)
+. $(librelib blacklist)
+
+# DBs = pacman DataBases
+
+# This replaces two scripts:
+#  - abslibre : imported ABS tree from Arch
+#  - db-sync  : imported pacman DBs from Arch
+
+# The flow here is:
+#  1. "${IMPORTDIR}/cache/${name}/dbs/" # Download the pacman databases
+#  2. "${IMPORTDIR}/cache/${name}/abs/" # Download the ABS tree
+#  3. "${IMPORTDIR}/clean/${name}/dbs/" # Run the pacman DBs through the blacklist
+#  4. Download all the package files mentioned in "clean/${name/dbs/"
+#  5. "${STAGING}-importer-${tag}" Copy all the package files we just downloaded to here
+#  6. Run db-update on "${STAGING}-importer-${tag}"
+
+SYNCARGS='-mrtvlH --no-motd --no-p --no-o --no-g'
+
+main() {
+	blacklist-update
+
+	local importStr
+	for importStr in "${imports[@]}"; do
+		local importAry=($importStr)
+		local name=${importAry[0]}
+		local pkgmirror=${importAry[1]}
+		local absmirror=${importAry[2]}
+		local tags=("${importAry[@]:3}")
+
+		msg "Fetching remote package source: %s" "$name"
+		fetch_dbs "$name" "$pkgmirror"
+		fetch_abs "$name" "$absmirror" "${tags[@]}"
+		msg "Filtering blacklisted packages from remote package source: %s" "$name"
+		clean_dbs "$name" "${tags[@]}"
+		msg "Publishing changes from remote package source: %s" "$name"
+		publish "$name" "${tags[@]}"
+	done
+	return $r
+}
+
+fetch_dbs() {
+	local name=$1
+	local pkgmirror=$2
+
+	msg2 'Synchronizing package databases...'
+
+	mkdir -p -- "${IMPORTDIR}/cache/${name}/dbs"
+	# Grab just the .db files from $pkgmirror
+	rsync $SYNCARGS --delete-after \
+		--include="*/" \
+		--include="*.db" \
+		--include="*${DBEXT}" \
+		--exclude="*" \
+		"rsync://${pkgmirror}/" "${IMPORTDIR}/cache/${name}/dbs"
+}
+
+fetch_abs() {
+	local name=$1
+	local absmirror=$2
+	local tags=("${@:3}")
+
+	local _HOME=$HOME
+
+	local absroot
+
+	# Sync the ABS tree from $absmirror
+	local arch
+	for arch in $(list_arches "${tags[@]}"); do
+		msg2 'Synchronizing %s ABS tree...' "$arch"
+
+		absroot="${IMPORTDIR}/cache/${name}/abs/${arch}"
+		mkdir -p -- "$absroot"
+
+		# Configure `abs` for this mirror
+		export HOME="${IMPORTDIR}/homes/${name}/${arch}"
+		mkdir -p -- "$HOME"
+		{
+			printf "ABSROOT='%s'\n" "$absroot"
+			printf "SYNCSERVER='%s'\n" "$absmirror"
+			printf "ARCH='%s'\n" "$arch"
+			printf 'REPOS=(\n'
+			list_repos "$arch" "${tags[@]}"
+			printf ')\n'
+		} > ~/.abs.conf
+
+		# Run `abs`
+		abs
+	done
+
+	export HOME=$_HOME
+}
+
+clean_dbs() {
+	local name=$1
+	local tags=("${@:2}")
+
+	rm -rf -- "${IMPORTDIR}/clean/$name"
+
+	local tag
+	for tag in "${tags[@]}"; do
+		msg2 'Creating clean version of %s package database...' "$tag"
+
+		local cache="${IMPORTDIR}/cache/$name/dbs/$(db_file "$tag")"
+		local clean="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")"
+		install -Dm644 "$cache" "$clean"
+
+		repo-remove "$clean" $(blacklist-cat|blacklist-get-pkg)
+	done
+}
+
+fetch_pkgs() {
+	local name=$1
+	local tags=("${@:2}")
+
+	local repo arch dbfile whitelist
+
+	local tag
+	for tag in "${tags[@]}"; do
+		msg2 'Syncronizing package files for %s...' "$tag"
+		repo=${tag%-*}
+		arch=${tag##*-}
+
+		dbfile="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")"
+		whitelist="${IMPORTDIR}/clean/$name/dbs/$tag.whitelist"
+
+		list_pkgs "$dbfile" > "$whitelist"
+
+		# fetch the architecture-specific packages
+		rsync $SYNCARGS --delete-after --delete-excluded \
+			--delay-updates \
+			--include-from=<(sed "s|\$|-$arch.tar.?z|" "$whitelist") \
+			--exclude='*' \
+			"rsync://${pkgmirror}/$(db_dir "$tag")/" \
+			"${IMPORTDIR}/clean/${name}/pkgs/${tag}/"
+
+		# fetch the architecture-independent packages
+		rsync $SYNCARGS --delete-after --delete-excluded \
+			--delay-updates \
+			--include-from=<(sed "s|\$|-any.tar.?z|" "$whitelist") \
+			--exclude='*' \
+			"rsync://${pkgmirror}/$(db_dir "$tag")/" \
+			"${IMPORTDIR}/clean/${name}/pkgs/${repo}-any/"
+	done
+}
+
+publish() {
+	local name=$1
+	local tags=("${@:2}")
+
+	local tag
+	for tag in "${tags[@]}"; do
+		msg2 'Publishing changes to %s...' "$tag"
+		publish_tag "$name" "$tag"
+	done
+}
+
+publish_tag() {
+	local name=$1
+	local tag=$2
+
+	local repo=${tag%-*}
+	local arch=${tag##*-}
+	local dir="${IMPORTDIR}/clean/${name}/pkgs/${tag}"
+
+	local found
+	local error=false
+	local files=()
+
+	local pkgid pkgarch
+	for pkgid in $(list_added_pkgs "$name" "$tag"); do
+		found=false
+
+		for pkgarch in "${arch}" any; do
+			file="${dir}/${pkgid}-${arch}".pkg.tar.?z
+			if ! $found && [[ -r $file ]]; then
+				files+=("$file")
+				found=true
+			fi
+		done
+
+		if ! $found; then
+			error 'Could not find package file for %s' "$pkgid"
+			error=true
+		fi
+	done
+
+	if $error; then
+		error 'Quitting...'
+		return 1
+	fi
+
+	local _STAGING=$STAGING
+	export STAGING="$STAGING-importer-$tag"
+
+	mkdir -p -- "${STAGING}/${repo}"
+	cp -a -- "${files[@]}" "${STAGING}/${repo}/"
+	db-update
+
+	# XXX: db-remove wants pkgbase, not pkgname
+	db-remove "$repo" "$arch" $(list_removed_pkgs "$name" "$tag")
+
+	STAGING=$_STAGING
+}
+
+################################################################################
+
+# Usage: list_arches repo-arch...
+# Returns a list of the architectures mentioned in a list of "repo-arch" pairs.
+list_arches() {
+	local tags=("$@")
+	printf '%s\n' "${tags[@]##*-}" | sort -u
+}
+
+# Usage: list_repos arch repo-arch...
+# Returns a list of all the repositories mentioned for a given architecture in a
+# list of "repo-arch" pairs.
+list_repos() {
+	local arch=$1
+	local tags=("${@:2}")
+	printf '%s\n' "${tags[@]}" | sed -n "s/-$arch\$//p"
+}
+
+# Usage: db_dir repo-arch
+db_dir() {
+	local tag=$1
+	local repo=${tag%-*}
+	local arch=${tag##*-}
+	echo "${repo}/os/${arch}"
+}
+
+# Usage; db_file repo-arch
+db_file() {
+	local tag=$1
+	local repo=${tag%-*}
+	local arch=${tag##*-}
+	echo "${repo}/os/${arch}/${repo}${DBEXT}"
+}
+
+# Usage: list_pkgs dbfile
+# Prints "$pkgname-$(get_full_version "$pkgname")" for every package in $dbfile
+list_pkgs() {
+	local dbfile=$1
+	bsdtar tf "$dbfile" | cut -d/ -f1
+}
+
+# Usage: list_pkgs | sep_ver
+# Separates the pkgname from the version (replaces the '-' with ' ') for the
+# list provided on stdin.
+sep_ver() {
+	sed -r 's/-([^-]*-[^-]*)$/ \1/'
+}
+
+# Usage: list_removed_pkgs importsrc repo-arch
+# Prints "$pkgname-$(get_full_version "$pkgname")" for every removed package.
+list_removed_pkgs() {
+	local name=$1
+	local tag=$2
+
+	local old="${FTP_BASE}/$(db_file "$tag")"
+	local new="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")"
+
+	# make a list of:
+	# pkgname oldver[ newver]
+	# It will include removed or updated packages (changed packages)
+	join -a1 \
+		<(list_pkgs "$old"|sep_ver|sort) \
+		<(list_pkgs "$new"|sep_ver|sort)
+	| grep -v ' .* ' # remove updated packages
+	| sed 's/ /-/' # re-combine the pkgname and version
+}
+
+# Usage: list_added_pkgs importsrc repo-arch
+# slightly a misnomer; added and updated
+# Prints "$pkgname-$(get_full_version "$pkgname")" for every added or updated
+# package.
+list_added_pkgs() {
+	local name=$1
+	local tag=$2
+
+	local old="${FTP_BASE}/$(db_file "$tag")"
+	local new="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")"
+
+	comm -13 <(list_pkgs "$old") <(list_pkgs "$new")
+}
+
+main "$@"
-- 
cgit v1.2.3-2-g168b


From bcfc6c88a6e23ac15f08b37d55cf1f5378b4120e Mon Sep 17 00:00:00 2001
From: Luke Shumaker <LukeShu@sbcglobal.net>
Date: Tue, 31 Dec 2013 14:19:10 -0500
Subject: db-import: run `readlink -e` on $0

---
 db-import | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'db-import')

diff --git a/db-import b/db-import
index 7bc822f..79dcfd5 100755
--- a/db-import
+++ b/db-import
@@ -2,10 +2,10 @@
 # Imports Arch-like repos, running them through a blacklist
 # License: GPLv3
 
-. $(dirname $0)/config
-. $(dirname $0)/db-import.conf
-. $(librelib messages)
-. $(librelib blacklist)
+. "$(dirname "$(readlink -e "$0")")/config"
+. "$(dirname "$(readlink -e "$0")")/db-import.conf"
+. "$(librelib messages)"
+. "$(librelib blacklist)"
 
 # DBs = pacman DataBases
 
-- 
cgit v1.2.3-2-g168b


From 8b6653a2c6c7e55f2c312735a31e0f42d7260bbd Mon Sep 17 00:00:00 2001
From: Luke Shumaker <LukeShu@sbcglobal.net>
Date: Mon, 9 Dec 2013 03:20:17 -0500
Subject: clean up db-import

---
 db-import | 45 ++++++++++++++++++++-------------------------
 1 file changed, 20 insertions(+), 25 deletions(-)

(limited to 'db-import')

diff --git a/db-import b/db-import
index 79dcfd5..a8a073d 100755
--- a/db-import
+++ b/db-import
@@ -1,9 +1,10 @@
-#!/bin/bash -euE
+#!/bin/bash
+set -euE
 # Imports Arch-like repos, running them through a blacklist
 # License: GPLv3
 
-. "$(dirname "$(readlink -e "$0")")/config"
-. "$(dirname "$(readlink -e "$0")")/db-import.conf"
+. "$(dirname "$(readlink -e "$0")")/config"         # for: FTP_BASE DBEXT
+. "$(dirname "$(readlink -e "$0")")/db-import.conf" # for: IMPORTDIR IMPORTS
 . "$(librelib messages)"
 . "$(librelib blacklist)"
 
@@ -17,17 +18,18 @@
 #  1. "${IMPORTDIR}/cache/${name}/dbs/" # Download the pacman databases
 #  2. "${IMPORTDIR}/cache/${name}/abs/" # Download the ABS tree
 #  3. "${IMPORTDIR}/clean/${name}/dbs/" # Run the pacman DBs through the blacklist
-#  4. Download all the package files mentioned in "clean/${name/dbs/"
-#  5. "${STAGING}-importer-${tag}" Copy all the package files we just downloaded to here
-#  6. Run db-update on "${STAGING}-importer-${tag}"
+#  4. "${IMPORTDIR}/clean/${name}/pkgs/" # Download the pkg files mentioned in "clean/${name}/dbs/"
+#  5. "${IMPORTDIR}/staging/${tag}" # Copy all the package files we just downloaded to here
+#  6. Run `db-update on` with STAGING="${IMPORTDIR}/staging/${tag}"
 
+# generic arguments to pass to rsync, borrowed from `abs`
 SYNCARGS='-mrtvlH --no-motd --no-p --no-o --no-g'
 
 main() {
 	blacklist-update
 
 	local importStr
-	for importStr in "${imports[@]}"; do
+	for importStr in "${IMPORTS[@]}"; do
 		local importAry=($importStr)
 		local name=${importAry[0]}
 		local pkgmirror=${importAry[1]}
@@ -39,6 +41,7 @@ main() {
 		fetch_abs "$name" "$absmirror" "${tags[@]}"
 		msg "Filtering blacklisted packages from remote package source: %s" "$name"
 		clean_dbs "$name" "${tags[@]}"
+		fetch_pkgs "$name" "${tags[@]}"
 		msg "Publishing changes from remote package source: %s" "$name"
 		publish "$name" "${tags[@]}"
 	done
@@ -66,8 +69,7 @@ fetch_abs() {
 	local absmirror=$2
 	local tags=("${@:3}")
 
-	local _HOME=$HOME
-
+	local fake_home
 	local absroot
 
 	# Sync the ABS tree from $absmirror
@@ -79,8 +81,8 @@ fetch_abs() {
 		mkdir -p -- "$absroot"
 
 		# Configure `abs` for this mirror
-		export HOME="${IMPORTDIR}/homes/${name}/${arch}"
-		mkdir -p -- "$HOME"
+		fake_home="${IMPORTDIR}/homes/${name}/${arch}"
+		mkdir -p -- "$fake_home"
 		{
 			printf "ABSROOT='%s'\n" "$absroot"
 			printf "SYNCSERVER='%s'\n" "$absmirror"
@@ -88,13 +90,11 @@ fetch_abs() {
 			printf 'REPOS=(\n'
 			list_repos "$arch" "${tags[@]}"
 			printf ')\n'
-		} > ~/.abs.conf
+		} > "${fake_home}/.abs.conf"
 
 		# Run `abs`
-		abs
+		HOME=$fake_home abs
 	done
-
-	export HOME=$_HOME
 }
 
 clean_dbs() {
@@ -111,7 +111,7 @@ clean_dbs() {
 		local clean="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")"
 		install -Dm644 "$cache" "$clean"
 
-		repo-remove "$clean" $(blacklist-cat|blacklist-get-pkg)
+		blacklist-cat | blacklist-get-pkg | xargs -d '\n' repo-remove "$clean"
 	done
 }
 
@@ -196,17 +196,12 @@ publish_tag() {
 		return 1
 	fi
 
-	local _STAGING=$STAGING
-	export STAGING="$STAGING-importer-$tag"
-
-	mkdir -p -- "${STAGING}/${repo}"
-	cp -a -- "${files[@]}" "${STAGING}/${repo}/"
-	db-update
+	mkdir -p -- "${IMPORTDIR}/staging/${tag}/${repo}"
+	cp -al -- "${files[@]}" "${IMPORTDIR}/staging/${tag}/${repo}/"
+	STAGING="${IMPORTDIR}/staging/${tag}" db-update
 
 	# XXX: db-remove wants pkgbase, not pkgname
-	db-remove "$repo" "$arch" $(list_removed_pkgs "$name" "$tag")
-
-	STAGING=$_STAGING
+	list_removed_pkgs "$name" "$tag" | xargs -d '\n' db-remove "$repo" "$arch"
 }
 
 ################################################################################
-- 
cgit v1.2.3-2-g168b