From 06fbc63477803a7323e57fd3663937dd1be88d39 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 2 Nov 2012 21:56:05 -0400 Subject: rewrite pbs-absrepo-convert --- pbs-absrepo-convert | 152 +++++++++++++++++++++++++++++++++++++--------- pbs-absrepo-convert--list | 7 +++ 2 files changed, 129 insertions(+), 30 deletions(-) create mode 100755 pbs-absrepo-convert--list diff --git a/pbs-absrepo-convert b/pbs-absrepo-convert index 2482d44..16349ad 100755 --- a/pbs-absrepo-convert +++ b/pbs-absrepo-convert @@ -1,58 +1,150 @@ #!/bin/bash -e +export TMPDIR="`mktemp -d --tmpdir pbs-absrepo-convert.XXXXXXXXXX`" +cleanup() { + msg "$(gettext "Removing temporary files...")" + echo rm -rf "$TMPDIR" +} +trap cleanup EXIT + . /usr/bin/libremessages -cachedir="$(pbs-plumb-config get core.cachedir)" -sourcedir="$(pbs-plumb-config get core.sourcedir)" -tmpdir="`mktemp -d --tmpdir pbs-absrepo-convert.XXXXXXXXXX`" -export TMPDIR=$tmpdir +abort() { + echo + error "$(gettext "Aborting...")" + cleanup +} -cleanup() { - rm -rf "$tmpdir" +_collect-data-gc() { + local base=$1 + local files=`echo $base.new*` + cat -- $files 2>>/dev/null | sort -u > $base.tmp + rm -f -- $files + mv $base.tmp $base.new } -trap cleanup EXIT +collect-data-gc() {( + cd "$TMPDIR" + while [[ ! -f collect-data-filter-branch.done ]]; do + _collect-data-gc packages + _collect-data-gc repo-arch + sleep 10 + done + mv packages.new packages + mv repo-arch.new repo-arch +)} -main() { +## +# Usage: collect-data +# Assumptions: +# 1. Currently in a working copy of the repo +# 2. git branch "master" exists and is untouched +# 3. TMPDIR is set and exists +## +collect-data() { + msg "$(gettext "Collecting package data...")" + collect-data-gc & + git filter-branch --tree-filter pbs-absrepo-convert--list master + # notify collect-data-gc that filter-branch is done + touch "${TMPDIR}/collect-data-filter-branch.done" + # wait for collect-data-gc to finish + while [[ ! -f "${TMPDIR}/packages" ]]; do + sleep 10; + done + # extract some data from those files + < "${TMPDIR}/repo-arch" sed -e 's/.*/-/' -e '/^any$/d' | sort -u > "${TMPDIR}/arches" + { echo master; cat "${TMPDIR}/arches"; } > "${TMPDIR}/arches.all" +} + +## +# Usage: convert-packages SOURCE +# Assumptions: +# 1. Currently in a working copy of the repo +# 2. TMPDIR is set, and collect-data has been run on it +## +convert-packages() { + msg "$(gettext "Converting packages...")" [[ $# != 1 ]] && { usage; return 1; } local source=$1 - # init ################################################################# - cd "${cachedir}/${source}.git" - git branch > "${tmpdir}/branches" - git clone "${cachedir}/${source}.git" "${tmpdir}/repo" + local ibranch + local obranch + local dir - cd "${tmpdir}/repo" - git checkout -b orig - git branch -D master - find */repos/* -type d -printf '%f\n' | sed -e 's/.*/-//' -e '/^any$/d' | sort -u > "${tmpdir}/architectures" + local count="$(wc -l < "${TMPDIR}/packages")" + cat -n "${TMPDIR}/packages" | while read n package; do + msg2 "$(gettext "(%d/%d) %s")" "$n" "$count" "$package" + if git checkout "packages/${package}" &>/dev/null; then + # common case (optimization) + ibranch=orig + obranch="packages/${package}" + dir=trunk + else + # general (uncommon) case + ibranch="packages/${package}" + obranch="packages/${source}/${package}" + dir="${package}/trunk" + fi + git checkout -b "$obranch" "$ibranch" + if [[ $ibranch != orig ]]; then + git branch -D "$ibranch" + fi + git filter-branch -f --prune-empty --subdirectory-filter "$dir" "$obranch" - # convert packages ##################################################### - msg "$(gettext "Converting packages...")" - package_count="$(wc -l "${tmpdir}/branches")" - package_count="${package_count%% *}" - cat "${tmpdir}/branches" | sed -n 's@^\s*packages/@@p' | cat -n | while read n package; do - msg2 "$(gettext "(%d/%d) %s")" "$n" "$package_count" "$package" - git checkout "packages/${package}" - git filter-branch -f --prune-empty --subdirectory-filter trunk "packages/${package}" - git log --pretty=format:'%T %H' > "${tmpdir}/packages-${package}.commits" + git log --pretty=format:'%T %H' > "${TMPDIR}/packages-${package}.commits" done +} - # convert architectures ################################################ - for arch in master $(cat "${tmpdir}/architectures"); do - msg "$(gettext "Creating architecture %s...")" "$arch" +## +# Usage: convert-arches +# Assumptions: +# 1. Currently in a working copy of the repo +# 2. TMPDIR is set, and collect-data has been run on it +# 3. ${TMPDIR}/packages-PACKAGE.commits exists for every package +## +convert-arches() { + msg "$(gettext "Converting architectures...")" + + local count="$(wc -l < "${TMPDIR}/arches.all")" + cat -n "${TMPDIR}/arches.all" | while read n arch; do + msg2 "$(gettext "(%d/%d) %s")" "$n" "$count" "$arch" git checkout orig git checkout -b "$arch" - git filter-branch --tree-filter "pbs-absrepo-convert--treefilter $arch" "$arch" + git filter-branch -f --tree-filter \ + "pbs-absrepo-convert--treefilter $arch" "$arch" done +} + +main() { + trap abort EXIT + [[ $# != 1 ]] && { usage; return 1; } + local source=$1 + + local cachedir="$(pbs-plumb-config get core.cachedir)" + local sourcedir="$(pbs-plumb-config get core.sourcedir)" + + # init ################################################################# + msg "$(gettext "Creating working copy...")" + git clone "${cachedir}/${source}.git" "${TMPDIR}/repo" + cd "${TMPDIR}/repo" + git checkout -b orig + git checkout master + collect-data + git branch -D master + + # main ################################################################# + + convert-packages "$source" + convert-arches "$source" # save results ######################################################### msg "$(gettext "Copying into source directory...")" [[ -d "$sourcedir" ]] || mkdir -p "$sourcedir" - git clone --mirror "${tmpdir}/repo" "${sourcedir}/${source}.new.git" + git clone --mirror "${TMPDIR}/repo" "${sourcedir}/${source}.new.git" [[ -d "${sourcedir}/${source}.git" ]] && mv "${sourcedir}/${source}"{,.old}.git mv "${sourcedir}/${source}"{.new,}.git [[ -d "${sourcedir}/${source}.old.git" ]] && rm -rf "${sourcedir}/${source}".old.git + trap cleanup EXIT } main "$@" diff --git a/pbs-absrepo-convert--list b/pbs-absrepo-convert--list new file mode 100755 index 0000000..527568a --- /dev/null +++ b/pbs-absrepo-convert--list @@ -0,0 +1,7 @@ +#!/bin/bash + +ls -1 > "${TMPDIR}/packages.tmp.${GIT_COMMIT}" +find */repos/* -type d -printf '%f\n' >> "${TMPDIR}/repo-arch.tmp.${GIT_COMMIT}" + +mv "${TMPDIR}/packages.tmp.${GIT_COMMIT}" "${TMPDIR}/packages.new.${GIT_COMMIT}" +mv "${TMPDIR}/repo-arch.tmp.${GIT_COMMIT}" "${TMPDIR}/repo-arch.new.${GIT_COMMIT}" -- cgit v1.1-4-g5e80