From 0e9dc9f69e290fe352de7b2bbd288890a78eced5 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 25 Aug 2010 11:30:42 -0500 Subject: Add update-web-db cronjob This has been untracked by version control for a while on gerolde in both the cron-jobs directory and in /etc/cron.hourly/. Add it here so we can make changes and know what is going on. This is an improved script over what we currently have. It is one script instead of two, and it does things a little smarter with the logging. Signed-off-by: Dan McGee --- cron-jobs/update-web-db | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100755 cron-jobs/update-web-db (limited to 'cron-jobs/update-web-db') diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db new file mode 100755 index 0000000..8631a04 --- /dev/null +++ b/cron-jobs/update-web-db @@ -0,0 +1,47 @@ +#!/bin/bash + +# run at nice 5. it can churn quite a bit of cpu after all. +renice +5 -p $$ > /dev/null + +# setup paths +SPATH="/srv/http/archweb" +ARCHES="i686 x86_64" +# having "more important repos" last should make [core] trickle to the top of +# the updates list each hour rather than being overwhelmed by big [extra] and +# [community] updates +REPOS="community-testing multilib community extra testing core" +LOGOUT="/tmp/archweb_update.log" + +# figure out what operation to perform +cmd="$(basename $0)" +if [[ $cmd != "update-web-db" && $cmd != "update-web-files-db" ]]; then + echo "Invalid command name '$cmd' specified!" + exit 1 +fi + +echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" + +## do the dirty, then output the result to a file. +cd $SPATH +for arch in ${ARCHES}; do + for repo in ${REPOS}; do + echo "Updating ${repo}-${arch}" + case "$cmd" in + update-web-db) + dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.db.tar.gz" + flags="" ;; + update-web-files-db) + dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.files.tar.gz" + flags="--filesonly" ;; + esac + ./manage.py reporead ${flags} ${arch} ${dbfile} + echo "" + done +done >> "${LOGOUT}" 2>&1 + +echo "" >> "${LOGOUT}" + +# rotate the file if it is getting big (> 10M), overwriting any old backup +if [[ $(stat -c%s update-web-db) -gt 10485760 ]]; then + mv "${LOGOUT}" "${LOGOUT}.old" +fi -- cgit v1.2.3-2-g168b From 960917cdb0b23163029dc81e52f295e3281492e1 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 25 Aug 2010 11:38:10 -0500 Subject: Fix a few issues with update-web-db Signed-off-by: Dan McGee --- cron-jobs/update-web-db | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'cron-jobs/update-web-db') diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 8631a04..4d1519c 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -25,7 +25,6 @@ echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" cd $SPATH for arch in ${ARCHES}; do for repo in ${REPOS}; do - echo "Updating ${repo}-${arch}" case "$cmd" in update-web-db) dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.db.tar.gz" @@ -34,14 +33,17 @@ for arch in ${ARCHES}; do dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.files.tar.gz" flags="--filesonly" ;; esac - ./manage.py reporead ${flags} ${arch} ${dbfile} - echo "" + if [[ -f $dbfile ]]; then + echo "Updating ${repo}-${arch}" + ./manage.py reporead ${flags} ${arch} ${dbfile} + echo "" + fi done -done >> "${LOGOUT}" 2>&1 +done >> "${LOGOUT}" 2>&1 echo "" >> "${LOGOUT}" # rotate the file if it is getting big (> 10M), overwriting any old backup -if [[ $(stat -c%s update-web-db) -gt 10485760 ]]; then +if [[ $(stat -c%s "${LOGOUT}") -gt 10485760 ]]; then mv "${LOGOUT}" "${LOGOUT}.old" fi -- cgit v1.2.3-2-g168b From 1d6eb15bcc4c355edef1d6c249e9c67bca7d0404 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 25 Aug 2010 11:39:16 -0500 Subject: update-web-db: reformat to coding style Signed-off-by: Dan McGee --- cron-jobs/update-web-db | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) (limited to 'cron-jobs/update-web-db') diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 4d1519c..3504813 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -15,8 +15,8 @@ LOGOUT="/tmp/archweb_update.log" # figure out what operation to perform cmd="$(basename $0)" if [[ $cmd != "update-web-db" && $cmd != "update-web-files-db" ]]; then - echo "Invalid command name '$cmd' specified!" - exit 1 + echo "Invalid command name '$cmd' specified!" + exit 1 fi echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" @@ -25,19 +25,19 @@ echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" cd $SPATH for arch in ${ARCHES}; do for repo in ${REPOS}; do - case "$cmd" in - update-web-db) - dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.db.tar.gz" - flags="" ;; - update-web-files-db) - dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.files.tar.gz" - flags="--filesonly" ;; - esac - if [[ -f $dbfile ]]; then - echo "Updating ${repo}-${arch}" - ./manage.py reporead ${flags} ${arch} ${dbfile} - echo "" - fi + case "$cmd" in + update-web-db) + dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.db.tar.gz" + flags="" ;; + update-web-files-db) + dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.files.tar.gz" + flags="--filesonly" ;; + esac + if [[ -f $dbfile ]]; then + echo "Updating ${repo}-${arch}" + ./manage.py reporead ${flags} ${arch} ${dbfile} + echo "" + fi done done >> "${LOGOUT}" 2>&1 @@ -45,5 +45,7 @@ echo "" >> "${LOGOUT}" # rotate the file if it is getting big (> 10M), overwriting any old backup if [[ $(stat -c%s "${LOGOUT}") -gt 10485760 ]]; then - mv "${LOGOUT}" "${LOGOUT}.old" + mv "${LOGOUT}" "${LOGOUT}.old" fi + +# vim: set ts=4 sw=4 et ft=sh: -- cgit v1.2.3-2-g168b From 3c07e2e3ff951242a5d9fc124c6937e75e961da2 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 25 Aug 2010 11:39:57 -0500 Subject: update-web-db: flip the arch/repo loops This will really make sure [core] packages flow to the top of the list, regardless of architecture. They should anyway, but anything that was only updated in the first architecture updated could get artificially pushed down before. Signed-off-by: Dan McGee --- cron-jobs/update-web-db | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cron-jobs/update-web-db') diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 3504813..8f91ca2 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -23,8 +23,8 @@ echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" ## do the dirty, then output the result to a file. cd $SPATH -for arch in ${ARCHES}; do - for repo in ${REPOS}; do +for repo in ${REPOS}; do + for arch in ${ARCHES}; do case "$cmd" in update-web-db) dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.db.tar.gz" -- cgit v1.2.3-2-g168b From 8a5f6e5542590edb94499bd6f36ba3787efbc2ce Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 12 Sep 2010 09:50:44 -0500 Subject: Add multilib-testing to web update script Signed-off-by: Dan McGee --- cron-jobs/update-web-db | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs/update-web-db') diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 8f91ca2..8b74a3a 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -9,7 +9,7 @@ ARCHES="i686 x86_64" # having "more important repos" last should make [core] trickle to the top of # the updates list each hour rather than being overwhelmed by big [extra] and # [community] updates -REPOS="community-testing multilib community extra testing core" +REPOS="community-testing multilib-testing multilib community extra testing core" LOGOUT="/tmp/archweb_update.log" # figure out what operation to perform -- cgit v1.2.3-2-g168b From 44c24c5d8e98eb51c8612dfaaac3ddd2ce0337dd Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 16 Sep 2010 18:50:53 -0500 Subject: Move output redirection Signed-off-by: Dan McGee --- cron-jobs/update-web-db | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'cron-jobs/update-web-db') diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 8b74a3a..275676e 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -34,12 +34,12 @@ for repo in ${REPOS}; do flags="--filesonly" ;; esac if [[ -f $dbfile ]]; then - echo "Updating ${repo}-${arch}" - ./manage.py reporead ${flags} ${arch} ${dbfile} + echo "Updating ${repo}-${arch}" >> "${LOGOUT}" + ./manage.py reporead ${flags} ${arch} ${dbfile} >> "${LOGOUT}" 2>&1 echo "" fi done -done >> "${LOGOUT}" 2>&1 +done echo "" >> "${LOGOUT}" -- cgit v1.2.3-2-g168b From 7711a02ed5ce1be7f5f0cb572a1efc4c29b6dfe0 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 11 Oct 2010 12:28:52 -0500 Subject: Fix missing output redirection Signed-off-by: Dan McGee --- cron-jobs/update-web-db | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cron-jobs/update-web-db') diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 275676e..3f4a2fc 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -36,7 +36,7 @@ for repo in ${REPOS}; do if [[ -f $dbfile ]]; then echo "Updating ${repo}-${arch}" >> "${LOGOUT}" ./manage.py reporead ${flags} ${arch} ${dbfile} >> "${LOGOUT}" 2>&1 - echo "" + echo "" >> "${LOGOUT}" fi done done -- cgit v1.2.3-2-g168b From b9dea6f306df1ffb051f0c7239f416a34f93a71a Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 25 Nov 2010 18:42:19 +0100 Subject: update-web-db: Lock db and script * usage of db-functions * Lock the db and work on a copy to avoid long lock times * Lock the script itself to avoid multiple calls * Adjust to coding style to other dbscripts --- cron-jobs/update-web-db | 72 +++++++++++++++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 26 deletions(-) (limited to 'cron-jobs/update-web-db') diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 3f4a2fc..8760fc6 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -1,51 +1,71 @@ #!/bin/bash -# run at nice 5. it can churn quite a bit of cpu after all. -renice +5 -p $$ > /dev/null +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" # setup paths SPATH="/srv/http/archweb" -ARCHES="i686 x86_64" # having "more important repos" last should make [core] trickle to the top of # the updates list each hour rather than being overwhelmed by big [extra] and # [community] updates -REPOS="community-testing multilib-testing multilib community extra testing core" +REPOS=('community-testing' 'multilib-testing' 'multilib' 'community' 'extra' 'testing' 'core') LOGOUT="/tmp/archweb_update.log" # figure out what operation to perform cmd="$(basename $0)" if [[ $cmd != "update-web-db" && $cmd != "update-web-files-db" ]]; then - echo "Invalid command name '$cmd' specified!" - exit 1 + die "Invalid command name '$cmd' specified!" fi +script_lock + +# run at nice 5. it can churn quite a bit of cpu after all. +renice +5 -p $$ > /dev/null + echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" -## do the dirty, then output the result to a file. -cd $SPATH -for repo in ${REPOS}; do - for arch in ${ARCHES}; do - case "$cmd" in - update-web-db) - dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.db.tar.gz" - flags="" ;; - update-web-files-db) - dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.files.tar.gz" - flags="--filesonly" ;; - esac - if [[ -f $dbfile ]]; then - echo "Updating ${repo}-${arch}" >> "${LOGOUT}" - ./manage.py reporead ${flags} ${arch} ${dbfile} >> "${LOGOUT}" 2>&1 - echo "" >> "${LOGOUT}" - fi - done +case "$cmd" in + update-web-db) + dbfileext="${DBEXT}" + flags="" + ;; + update-web-files-db) + dbfileext="${FILESEXT}" + flags="--filesonly" + ;; +esac + +# Lock the repos and get a copy of the db files to work on +for repo in ${REPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_lock ${repo} ${arch} || exit 1 + dbfile="/srv/ftp/${repo}/os/${arch}/${repo}${dbfileext}" + if [ -f "${dbfile}" ]; then + mkdir -p "${WORKDIR}/${repo}/${arch}" + cp "${dbfile}" "${WORKDIR}/${repo}/${arch}/${repo}${dbfileext}" + fi + repo_unlock ${repo} ${arch} + done done +# Run reporead on our db copy +pushd $SPATH >/dev/null +for repo in ${REPOS[@]}; do + for arch in ${ARCHES[@]}; do + dbcopy="${WORKDIR}/${repo}/${arch}/${repo}${dbfileext}" + if [ -f "${dbcopy}" ]; then + echo "Updating ${repo}-${arch}" >> "${LOGOUT}" + ./manage.py reporead ${flags} ${arch} "${dbcopy}" >> "${LOGOUT}" 2>&1 + echo "" >> "${LOGOUT}" + fi + done +done +popd >/dev/null echo "" >> "${LOGOUT}" # rotate the file if it is getting big (> 10M), overwriting any old backup if [[ $(stat -c%s "${LOGOUT}") -gt 10485760 ]]; then - mv "${LOGOUT}" "${LOGOUT}.old" + mv "${LOGOUT}" "${LOGOUT}.old" fi -# vim: set ts=4 sw=4 et ft=sh: +script_unlock -- cgit v1.2.3-2-g168b From 7519447709e1ac9a5b5edaa04b32c03ac09ee2ef Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 8 Jan 2011 12:02:11 -0600 Subject: archweb update: source virtualenv if it exists Signed-off-by: Dan McGee --- cron-jobs/update-web-db | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'cron-jobs/update-web-db') diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 8760fc6..6ced4c1 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -5,6 +5,8 @@ # setup paths SPATH="/srv/http/archweb" +ENVPATH="/srv/http/archweb-env/bin/activate" + # having "more important repos" last should make [core] trickle to the top of # the updates list each hour rather than being overwhelmed by big [extra] and # [community] updates @@ -24,6 +26,11 @@ renice +5 -p $$ > /dev/null echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" +# source our virtualenv if it exists +if [ -f "$ENVPATH" ]; then + . "$ENVPATH" +fi + case "$cmd" in update-web-db) dbfileext="${DBEXT}" -- cgit v1.2.3-2-g168b