rewrite pbs-absrepo-convert

2 files changed, 129 insertions, 30 deletions

diff --git a/pbs-absrepo-convert b/pbs-absrepo-convert
index 2482d44..16349ad 100755
--- a/pbs-absrepo-convert
+++ b/pbs-absrepo-convert
@@ -1,58 +1,150 @@
 #!/bin/bash -e
 
+export TMPDIR="`mktemp -d --tmpdir pbs-absrepo-convert.XXXXXXXXXX`"
+cleanup() {
+	msg "$(gettext "Removing temporary files...")"
+	echo rm -rf "$TMPDIR"
+}
+trap cleanup EXIT
+
 . /usr/bin/libremessages
 
-cachedir="$(pbs-plumb-config get core.cachedir)"
-sourcedir="$(pbs-plumb-config get core.sourcedir)"
-tmpdir="`mktemp -d --tmpdir pbs-absrepo-convert.XXXXXXXXXX`"
-export TMPDIR=$tmpdir
+abort() {
+	echo
+	error "$(gettext "Aborting...")"
+	cleanup
+}
 
-cleanup() {
-	rm -rf "$tmpdir"
+_collect-data-gc() {
+	local base=$1
+	local files=`echo $base.new*`
+	cat -- $files 2>>/dev/null | sort -u > $base.tmp
+	rm -f -- $files
+	mv $base.tmp $base.new
 }
 
-trap cleanup EXIT
+collect-data-gc() {(
+	cd "$TMPDIR"
+	while [[ ! -f collect-data-filter-branch.done ]]; do
+		_collect-data-gc packages
+		_collect-data-gc repo-arch
+		sleep 10
+	done
+	mv packages.new packages
+	mv repo-arch.new repo-arch
+)}
 
-main() {
+##
+# Usage: collect-data
+# Assumptions:
+#   1. Currently in a working copy of the repo
+#   2. git branch "master" exists and is untouched
+#   3. TMPDIR is set and exists
+##
+collect-data() {
+	msg "$(gettext "Collecting package data...")"
+	collect-data-gc &
+	git filter-branch --tree-filter pbs-absrepo-convert--list master
+	# notify collect-data-gc that filter-branch is done
+	touch "${TMPDIR}/collect-data-filter-branch.done"
+	# wait for collect-data-gc to finish
+	while [[ ! -f "${TMPDIR}/packages" ]]; do
+		sleep 10;
+	done
+	# extract some data from those files
+	< "${TMPDIR}/repo-arch" sed -e 's/.*/-/' -e '/^any$/d' | sort -u > "${TMPDIR}/arches"
+	{ echo master; cat "${TMPDIR}/arches"; } > "${TMPDIR}/arches.all"
+}
+
+##
+# Usage: convert-packages SOURCE
+# Assumptions:
+#   1. Currently in a working copy of the repo
+#   2. TMPDIR is set, and collect-data has been run on it
+##
+convert-packages() {
+	msg "$(gettext "Converting packages...")"
 	[[ $# != 1 ]] && { usage; return 1; }
 	local source=$1
 
-	# init #################################################################
-	cd "${cachedir}/${source}.git"
-	git branch > "${tmpdir}/branches"
-	git clone "${cachedir}/${source}.git" "${tmpdir}/repo"
+	local ibranch
+	local obranch
+	local dir
 
-	cd "${tmpdir}/repo"
-	git checkout -b orig
-	git branch -D master
-	find */repos/* -type d -printf '%f\n' | sed -e 's/.*/-//' -e '/^any$/d' | sort -u > "${tmpdir}/architectures"
+	local count="$(wc -l < "${TMPDIR}/packages")"
+	cat -n "${TMPDIR}/packages" | while read n package; do
+		msg2 "$(gettext "(%d/%d) %s")" "$n" "$count" "$package"
+		if git checkout "packages/${package}" &>/dev/null; then
+			# common case (optimization)
+			ibranch=orig
+			obranch="packages/${package}"
+			dir=trunk
+		else
+			# general (uncommon) case
+			ibranch="packages/${package}"
+			obranch="packages/${source}/${package}"
+			dir="${package}/trunk"
+		fi
+		git checkout -b "$obranch" "$ibranch"
+		if [[ $ibranch != orig ]]; then
+			git branch -D "$ibranch"
+		fi
+		git filter-branch -f --prune-empty --subdirectory-filter "$dir" "$obranch"
 
-	# convert packages #####################################################
-	msg "$(gettext "Converting packages...")"
-	package_count="$(wc -l "${tmpdir}/branches")"
-	package_count="${package_count%% *}"
-	cat "${tmpdir}/branches" | sed -n 's@^\s*packages/@@p' | cat -n | while read n package; do
-		msg2 "$(gettext "(%d/%d) %s")" "$n" "$package_count" "$package"
-		git checkout "packages/${package}"
-		git filter-branch -f --prune-empty --subdirectory-filter trunk "packages/${package}"
-		git log --pretty=format:'%T %H' > "${tmpdir}/packages-${package}.commits"
+		git log --pretty=format:'%T %H' > "${TMPDIR}/packages-${package}.commits"
 	done
+}
 
-	# convert architectures ################################################
-	for arch in master $(cat "${tmpdir}/architectures"); do
-		msg "$(gettext "Creating architecture %s...")" "$arch"
+##
+# Usage: convert-arches
+# Assumptions:
+#   1. Currently in a working copy of the repo
+#   2. TMPDIR is set, and collect-data has been run on it
+#   3. ${TMPDIR}/packages-PACKAGE.commits exists for every package
+##
+convert-arches() {
+	msg "$(gettext "Converting architectures...")"
+
+	local count="$(wc -l < "${TMPDIR}/arches.all")"
+	cat -n "${TMPDIR}/arches.all" | while read n arch; do
+		msg2 "$(gettext "(%d/%d) %s")" "$n" "$count" "$arch"
 		git checkout orig
 		git checkout -b "$arch"
-		git filter-branch --tree-filter "pbs-absrepo-convert--treefilter $arch" "$arch"
+		git filter-branch -f --tree-filter \
+			"pbs-absrepo-convert--treefilter $arch" "$arch"
 	done
+}
+
+main() {
+	trap abort EXIT
+	[[ $# != 1 ]] && { usage; return 1; }
+	local source=$1
+
+	local cachedir="$(pbs-plumb-config get core.cachedir)"
+	local sourcedir="$(pbs-plumb-config get core.sourcedir)"
+
+	# init #################################################################
+	msg "$(gettext "Creating working copy...")"
+	git clone "${cachedir}/${source}.git" "${TMPDIR}/repo"
+	cd "${TMPDIR}/repo"
+	git checkout -b orig
+	git checkout master
+	collect-data
+	git branch -D master
+
+	# main #################################################################
+
+	convert-packages "$source"
+	convert-arches "$source"
 
 	# save results #########################################################
 	msg "$(gettext "Copying into source directory...")"
 	[[ -d "$sourcedir" ]] || mkdir -p "$sourcedir"
-	git clone --mirror "${tmpdir}/repo" "${sourcedir}/${source}.new.git"
+	git clone --mirror "${TMPDIR}/repo" "${sourcedir}/${source}.new.git"
 	[[ -d "${sourcedir}/${source}.git" ]] && mv "${sourcedir}/${source}"{,.old}.git
 	mv "${sourcedir}/${source}"{.new,}.git
 	[[ -d "${sourcedir}/${source}.old.git" ]] && rm -rf "${sourcedir}/${source}".old.git
+	trap cleanup EXIT
 }
 
 main "$@"
diff --git a/pbs-absrepo-convert--list b/pbs-absrepo-convert--list
new file mode 100755
index 0000000..527568a
--- /dev/null
+++ b/pbs-absrepo-convert--list
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+ls -1 > "${TMPDIR}/packages.tmp.${GIT_COMMIT}"
+find */repos/* -type d -printf '%f\n' >> "${TMPDIR}/repo-arch.tmp.${GIT_COMMIT}"
+
+mv "${TMPDIR}/packages.tmp.${GIT_COMMIT}" "${TMPDIR}/packages.new.${GIT_COMMIT}"
+mv "${TMPDIR}/repo-arch.tmp.${GIT_COMMIT}" "${TMPDIR}/repo-arch.new.${GIT_COMMIT}"