diff options
Diffstat (limited to 'extra/lukeshu-xbs/db-import')
-rwxr-xr-x | extra/lukeshu-xbs/db-import | 288 |
1 files changed, 0 insertions, 288 deletions
diff --git a/extra/lukeshu-xbs/db-import b/extra/lukeshu-xbs/db-import deleted file mode 100755 index a8a073d..0000000 --- a/extra/lukeshu-xbs/db-import +++ /dev/null @@ -1,288 +0,0 @@ -#!/bin/bash -set -euE -# Imports Arch-like repos, running them through a blacklist -# License: GPLv3 - -. "$(dirname "$(readlink -e "$0")")/config" # for: FTP_BASE DBEXT -. "$(dirname "$(readlink -e "$0")")/db-import.conf" # for: IMPORTDIR IMPORTS -. "$(librelib messages)" -. "$(librelib blacklist)" - -# DBs = pacman DataBases - -# This replaces two scripts: -# - abslibre : imported ABS tree from Arch -# - db-sync : imported pacman DBs from Arch - -# The flow here is: -# 1. "${IMPORTDIR}/cache/${name}/dbs/" # Download the pacman databases -# 2. "${IMPORTDIR}/cache/${name}/abs/" # Download the ABS tree -# 3. "${IMPORTDIR}/clean/${name}/dbs/" # Run the pacman DBs through the blacklist -# 4. "${IMPORTDIR}/clean/${name}/pkgs/" # Download the pkg files mentioned in "clean/${name}/dbs/" -# 5. "${IMPORTDIR}/staging/${tag}" # Copy all the package files we just downloaded to here -# 6. Run `db-update on` with STAGING="${IMPORTDIR}/staging/${tag}" - -# generic arguments to pass to rsync, borrowed from `abs` -SYNCARGS='-mrtvlH --no-motd --no-p --no-o --no-g' - -main() { - blacklist-update - - local importStr - for importStr in "${IMPORTS[@]}"; do - local importAry=($importStr) - local name=${importAry[0]} - local pkgmirror=${importAry[1]} - local absmirror=${importAry[2]} - local tags=("${importAry[@]:3}") - - msg "Fetching remote package source: %s" "$name" - fetch_dbs "$name" "$pkgmirror" - fetch_abs "$name" "$absmirror" "${tags[@]}" - msg "Filtering blacklisted packages from remote package source: %s" "$name" - clean_dbs "$name" "${tags[@]}" - fetch_pkgs "$name" "${tags[@]}" - msg "Publishing changes from remote package source: %s" "$name" - publish "$name" "${tags[@]}" - done - return $r -} - -fetch_dbs() { - local name=$1 - local pkgmirror=$2 - - msg2 'Synchronizing package databases...' - - mkdir -p -- "${IMPORTDIR}/cache/${name}/dbs" - # Grab just the .db files from $pkgmirror - rsync $SYNCARGS --delete-after \ - --include="*/" \ - --include="*.db" \ - --include="*${DBEXT}" \ - --exclude="*" \ - "rsync://${pkgmirror}/" "${IMPORTDIR}/cache/${name}/dbs" -} - -fetch_abs() { - local name=$1 - local absmirror=$2 - local tags=("${@:3}") - - local fake_home - local absroot - - # Sync the ABS tree from $absmirror - local arch - for arch in $(list_arches "${tags[@]}"); do - msg2 'Synchronizing %s ABS tree...' "$arch" - - absroot="${IMPORTDIR}/cache/${name}/abs/${arch}" - mkdir -p -- "$absroot" - - # Configure `abs` for this mirror - fake_home="${IMPORTDIR}/homes/${name}/${arch}" - mkdir -p -- "$fake_home" - { - printf "ABSROOT='%s'\n" "$absroot" - printf "SYNCSERVER='%s'\n" "$absmirror" - printf "ARCH='%s'\n" "$arch" - printf 'REPOS=(\n' - list_repos "$arch" "${tags[@]}" - printf ')\n' - } > "${fake_home}/.abs.conf" - - # Run `abs` - HOME=$fake_home abs - done -} - -clean_dbs() { - local name=$1 - local tags=("${@:2}") - - rm -rf -- "${IMPORTDIR}/clean/$name" - - local tag - for tag in "${tags[@]}"; do - msg2 'Creating clean version of %s package database...' "$tag" - - local cache="${IMPORTDIR}/cache/$name/dbs/$(db_file "$tag")" - local clean="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")" - install -Dm644 "$cache" "$clean" - - blacklist-cat | blacklist-get-pkg | xargs -d '\n' repo-remove "$clean" - done -} - -fetch_pkgs() { - local name=$1 - local tags=("${@:2}") - - local repo arch dbfile whitelist - - local tag - for tag in "${tags[@]}"; do - msg2 'Syncronizing package files for %s...' "$tag" - repo=${tag%-*} - arch=${tag##*-} - - dbfile="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")" - whitelist="${IMPORTDIR}/clean/$name/dbs/$tag.whitelist" - - list_pkgs "$dbfile" > "$whitelist" - - # fetch the architecture-specific packages - rsync $SYNCARGS --delete-after --delete-excluded \ - --delay-updates \ - --include-from=<(sed "s|\$|-$arch.tar.?z|" "$whitelist") \ - --exclude='*' \ - "rsync://${pkgmirror}/$(db_dir "$tag")/" \ - "${IMPORTDIR}/clean/${name}/pkgs/${tag}/" - - # fetch the architecture-independent packages - rsync $SYNCARGS --delete-after --delete-excluded \ - --delay-updates \ - --include-from=<(sed "s|\$|-any.tar.?z|" "$whitelist") \ - --exclude='*' \ - "rsync://${pkgmirror}/$(db_dir "$tag")/" \ - "${IMPORTDIR}/clean/${name}/pkgs/${repo}-any/" - done -} - -publish() { - local name=$1 - local tags=("${@:2}") - - local tag - for tag in "${tags[@]}"; do - msg2 'Publishing changes to %s...' "$tag" - publish_tag "$name" "$tag" - done -} - -publish_tag() { - local name=$1 - local tag=$2 - - local repo=${tag%-*} - local arch=${tag##*-} - local dir="${IMPORTDIR}/clean/${name}/pkgs/${tag}" - - local found - local error=false - local files=() - - local pkgid pkgarch - for pkgid in $(list_added_pkgs "$name" "$tag"); do - found=false - - for pkgarch in "${arch}" any; do - file="${dir}/${pkgid}-${arch}".pkg.tar.?z - if ! $found && [[ -r $file ]]; then - files+=("$file") - found=true - fi - done - - if ! $found; then - error 'Could not find package file for %s' "$pkgid" - error=true - fi - done - - if $error; then - error 'Quitting...' - return 1 - fi - - mkdir -p -- "${IMPORTDIR}/staging/${tag}/${repo}" - cp -al -- "${files[@]}" "${IMPORTDIR}/staging/${tag}/${repo}/" - STAGING="${IMPORTDIR}/staging/${tag}" db-update - - # XXX: db-remove wants pkgbase, not pkgname - list_removed_pkgs "$name" "$tag" | xargs -d '\n' db-remove "$repo" "$arch" -} - -################################################################################ - -# Usage: list_arches repo-arch... -# Returns a list of the architectures mentioned in a list of "repo-arch" pairs. -list_arches() { - local tags=("$@") - printf '%s\n' "${tags[@]##*-}" | sort -u -} - -# Usage: list_repos arch repo-arch... -# Returns a list of all the repositories mentioned for a given architecture in a -# list of "repo-arch" pairs. -list_repos() { - local arch=$1 - local tags=("${@:2}") - printf '%s\n' "${tags[@]}" | sed -n "s/-$arch\$//p" -} - -# Usage: db_dir repo-arch -db_dir() { - local tag=$1 - local repo=${tag%-*} - local arch=${tag##*-} - echo "${repo}/os/${arch}" -} - -# Usage; db_file repo-arch -db_file() { - local tag=$1 - local repo=${tag%-*} - local arch=${tag##*-} - echo "${repo}/os/${arch}/${repo}${DBEXT}" -} - -# Usage: list_pkgs dbfile -# Prints "$pkgname-$(get_full_version "$pkgname")" for every package in $dbfile -list_pkgs() { - local dbfile=$1 - bsdtar tf "$dbfile" | cut -d/ -f1 -} - -# Usage: list_pkgs | sep_ver -# Separates the pkgname from the version (replaces the '-' with ' ') for the -# list provided on stdin. -sep_ver() { - sed -r 's/-([^-]*-[^-]*)$/ \1/' -} - -# Usage: list_removed_pkgs importsrc repo-arch -# Prints "$pkgname-$(get_full_version "$pkgname")" for every removed package. -list_removed_pkgs() { - local name=$1 - local tag=$2 - - local old="${FTP_BASE}/$(db_file "$tag")" - local new="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")" - - # make a list of: - # pkgname oldver[ newver] - # It will include removed or updated packages (changed packages) - join -a1 \ - <(list_pkgs "$old"|sep_ver|sort) \ - <(list_pkgs "$new"|sep_ver|sort) - | grep -v ' .* ' # remove updated packages - | sed 's/ /-/' # re-combine the pkgname and version -} - -# Usage: list_added_pkgs importsrc repo-arch -# slightly a misnomer; added and updated -# Prints "$pkgname-$(get_full_version "$pkgname")" for every added or updated -# package. -list_added_pkgs() { - local name=$1 - local tag=$2 - - local old="${FTP_BASE}/$(db_file "$tag")" - local new="${IMPORTDIR}/clean/$name/dbs/$(db_file "$tag")" - - comm -13 <(list_pkgs "$old") <(list_pkgs "$new") -} - -main "$@" |