From 8eafe04ec5eaf917d9e2d58993604c454f1a26d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?coadde=20=5BM=C3=A1rcio=20Alexandre=20Silva=20Delgado=5D?= Date: Wed, 26 Aug 2015 03:33:26 -0300 Subject: re-estructure dbscrips --- extra/coadde/cron-jobs/sourceballs | 151 +++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100755 extra/coadde/cron-jobs/sourceballs (limited to 'extra/coadde/cron-jobs/sourceballs') diff --git a/extra/coadde/cron-jobs/sourceballs b/extra/coadde/cron-jobs/sourceballs new file mode 100755 index 0000000..c12a128 --- /dev/null +++ b/extra/coadde/cron-jobs/sourceballs @@ -0,0 +1,151 @@ +#!/bin/bash + +dirname="$(dirname "$(readlink -e "$0")")" +. "${dirname}/../config" +. "${dirname}/../db-functions" +pushd "${WORKDIR}" >/dev/null + +script_lock + +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do + repo_lock "${repo}" "${arch}" || exit 1 + done +done + +#adjust the nice level to run at a lower priority +renice +10 -p $$ > /dev/null + +# Create a readable file for each repo with the following format +# - [ ] +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do + # Repo does not exist; skip it + if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + continue + fi + bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ + | awk '/^%NAME%/ { getline b }; + /^%BASE%/ { getline b }; + /^%VERSION%/ { getline v }; + /^%LICENSE%/,/^$/ { + if ( !/^%LICENSE%/ ) { l=l" "$0 } + }; + /^%ARCH%/ { + getline a; + printf "%s %s %s %s\n", b, v, a, l; + l=""; + }' + done | sort -u > "${WORKDIR}/db-${repo}" +done + +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do + repo_unlock "${repo}" "${arch}" + done +done + +# Create a list of all available source package file names +find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" + +# Check for all packages if we need to build a source package +for repo in "${PKGREPOS[@]}"; do + newpkgs=() + failedpkgs=() + while read line; do + pkginfo=("${line}") + pkgbase=${pkginfo[0]} + pkgver=${pkginfo[1]} + pkgarch=${pkginfo[2]} + pkglicense=("${pkginfo[@]:3}") + + # Should this package be skipped? + if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then + continue + fi + # Commenting out, we'll sourceball everything + # Check if the license or .force file does not enforce creating a source package +# if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then +# continue +# fi + # Store the expected file name of the source package + echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" + + # Build the source package if its not already there + if ! grep -Fqx "${pkgbase}-${pkgver}${SRCEXT}" "${WORKDIR}/available-src-pkgs"; then + # Check if we had failed before + if in_array "${pkgbase}-${pkgver}${SRCEXT}" "${failedpkgs[@]}"; then + continue + fi + + # Get the sources from xbs + mkdir -p -m0770 "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" + cp -a "$(xbs releasepath "${pkgbase}" "${repo}" "${pkgarch}")" \ + "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 + if [ $? -ge 1 ]; then + failedpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") + continue + fi + + # Build the actual source package + pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null + SRCPKGDEST=. makepkg --nocolor --allsource --ignorearch --skippgpcheck >"${WORKDIR}/${pkgbase}.log" 2>&1 + if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then + mv_acl "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}/${pkgbase}-${pkgver}${SRCEXT}" + # Avoid creating the same source package for every arch + echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" + newpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") + else + failedpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") + cat "${WORKDIR}/${pkgbase}.log" >> "${WORKDIR}/makepkg-fail.log" + fi + popd >/dev/null + fi + done < "${WORKDIR}/db-${repo}" + + if [ ${#newpkgs[@]} -ge 1 ]; then + msg "Adding source packages for [%s]..." "${repo}" + for new_pkg in "${newpkgs[@]}"; do + msg2 '%s' "${new_pkg}" + done + fi + if [ ${#failedpkgs[@]} -ge 1 ]; then + msg "Failed to create source packages for [%s]..." "${repo}" + for failed_pkg in "${failedpkgs[@]}"; do + msg2 '%s' "${failed_pkg}" + done + fi +done + +# Cleanup old source packages +cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" +cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort" +old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) + +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages..." + "${SOURCE_CLEANUP_DRYRUN}" && warning 'dry run mode is active' + for old_pkg in "${old_pkgs[@]}"; do + msg2 '%s' "${old_pkg}" + if ! "${SOURCE_CLEANUP_DRYRUN}"; then + mv_acl "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + fi + done +fi + +old_pkgs=($(find "${SOURCE_CLEANUP_DESTDIR}" -type f -name "*${SRCEXT}" -mtime +"${SOURCE_CLEANUP_KEEP}" -printf '%f\n')) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages from the cleanup directory..." + for old_pkg in "${old_pkgs[@]}"; do + msg2 '%s' "${old_pkg}" + "${SOURCE_CLEANUP_DRYRUN}" || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + done +fi + +if [ -f "${WORKDIR}/makepkg-fail.log" ]; then + msg "Log of failed packages" + cat "${WORKDIR}/makepkg-fail.log" +fi + +script_unlock -- cgit v1.2.3-2-g168b