From c9ad9623c80ddf2db2a734b4a7914ab9eebd84fb Mon Sep 17 00:00:00 2001 From: Joseph Graham Date: Sun, 1 Jun 2014 16:04:55 +0100 Subject: Added the script `make_repo_torrents' which makes torrents for all the packages. --- cron-jobs/make_repo_torrents | 81 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 cron-jobs/make_repo_torrents diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents new file mode 100644 index 0000000..d3c3df6 --- /dev/null +++ b/cron-jobs/make_repo_torrents @@ -0,0 +1,81 @@ +#! /bin/bash +# Copyright (C) 2014 Joseph Graham +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# This script depends on `mktorrent' + +username=$( id -un ) + +case "${username}" in + repo | root ) + true + ;; + * ) + echo "This script must be run as repo user or root user." + echo "ByeBye!" + exit 1 + ;; +esac + +# pacman doesn't support multiple different packages of the same name, +# so it's OK to just stuff all the torrents into a single directory. +torrent_location='/srv/http/repo/public/torrents/' +public_location='/srv/http/repo/public/' + +# Tracker announce URL +tracker='http://t67.eu:6969/announce' # t67.eu is run by Xylon + +# All mirrors go here +declare -a url_prefixes=('http://repo.parabolagnulinux.org/' 'https://parabola.goodgnus.com.ar/' 'http://mirror.yandex.ru/mirrors/parabola/' 'http://alfplayer.com/parabola/') + +cd "${torrent_location}" + +find "${public_location}" -name 'os' -type 'd' | +while read dir +do + find "${dir}" -name '*\.pkg\.tar\.xz' | + while read pkg + do + pkg_name="${pkg##*/}" + + if [[ -h "${pkg}" ]] # check if it's a symbolic link + then + # We get the target of the symlink + pkg=$( readlink -f "${pkg}" ) + fi + + if ! [[ -f "${torrent_location}${pkg_name}.torrent" ]] + then + # We need to make a comma seperated list of webseeds (this is passed + # as a single argument to mktorrent) + webseeds='' + + for prefix in "${url_prefixes[@]}" + do + webseeds+="${prefix}${pkg#${public_location}}," + done + + # There should not be a random comma at the end of the webseeds + webseeds="${webseeds%,}" + + mktorrent -a "${tracker}" "${pkg}" -w "${webseeds}" # "${torrent_location}" + fi + done +done + +if [[ "${username}" == root ]] +then + chown repo * +fi -- cgit v1.1-4-g5e80 From ee8da885dc5062001861e4355602a39ae972c32c Mon Sep 17 00:00:00 2001 From: Joseph Graham Date: Mon, 2 Jun 2014 12:48:47 +0100 Subject: Made it randomize the order of the webseed list for every torrent. --- cron-jobs/make_repo_torrents | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents index d3c3df6..dffd0dc 100644 --- a/cron-jobs/make_repo_torrents +++ b/cron-jobs/make_repo_torrents @@ -38,7 +38,25 @@ public_location='/srv/http/repo/public/' tracker='http://t67.eu:6969/announce' # t67.eu is run by Xylon # All mirrors go here -declare -a url_prefixes=('http://repo.parabolagnulinux.org/' 'https://parabola.goodgnus.com.ar/' 'http://mirror.yandex.ru/mirrors/parabola/' 'http://alfplayer.com/parabola/') +declare -a array=('http://repo.parabolagnulinux.org/' 'https://parabola.goodgnus.com.ar/' 'http://mirror.yandex.ru/mirrors/parabola/' 'http://alfplayer.com/parabola/') + +# I got this function from http://mywiki.wooledge.org/BashFAQ/026 . It +# shuffles an array. Uses a global array variable. Must be compact +# (not a sparse array). The array must be called `array'. +shuffle() { + local i tmp size max rand + + # $RANDOM % (i+1) is biased because of the limited range of $RANDOM + # Compensate by using a range which is a multiple of the array size. + size=${#array[*]} + max=$(( 32768 / size * size )) + + for ((i=size-1; i>0; i--)); do + while (( (rand=$RANDOM) >= max )); do :; done + rand=$(( rand % (i+1) )) + tmp=${array[i]} array[i]=${array[rand]} array[rand]=$tmp + done +} cd "${torrent_location}" @@ -62,7 +80,12 @@ do # as a single argument to mktorrent) webseeds='' - for prefix in "${url_prefixes[@]}" + # Randomize the order of the list of webseeds because I + # don't know if transmission might always use the one at + # the top otherwize. + shuffle + + for prefix in "${array[@]}" do webseeds+="${prefix}${pkg#${public_location}}," done -- cgit v1.1-4-g5e80 From 6979cf5d1e4a599fbabaed4fe643e9db0f0f871c Mon Sep 17 00:00:00 2001 From: Joseph Graham Date: Mon, 2 Jun 2014 12:54:47 +0100 Subject: Added encyclomundi's mirror to the webseeds array. --- cron-jobs/make_repo_torrents | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents index dffd0dc..246264d 100644 --- a/cron-jobs/make_repo_torrents +++ b/cron-jobs/make_repo_torrents @@ -38,7 +38,7 @@ public_location='/srv/http/repo/public/' tracker='http://t67.eu:6969/announce' # t67.eu is run by Xylon # All mirrors go here -declare -a array=('http://repo.parabolagnulinux.org/' 'https://parabola.goodgnus.com.ar/' 'http://mirror.yandex.ru/mirrors/parabola/' 'http://alfplayer.com/parabola/') +declare -a array=('http://repo.parabolagnulinux.org/' 'https://parabola.goodgnus.com.ar/' 'http://mirror.yandex.ru/mirrors/parabola/' 'http://alfplayer.com/parabola/' 'http://mirror.parlementum.net/') # I got this function from http://mywiki.wooledge.org/BashFAQ/026 . It # shuffles an array. Uses a global array variable. Must be compact -- cgit v1.1-4-g5e80 From ad79659b3057fd5afac3629e9dd53ea3b93fb58e Mon Sep 17 00:00:00 2001 From: Joseph Graham Date: Fri, 6 Jun 2014 12:29:21 +0100 Subject: Made it only give output on an error. --- cron-jobs/make_repo_torrents | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents index 246264d..797142e 100644 --- a/cron-jobs/make_repo_torrents +++ b/cron-jobs/make_repo_torrents @@ -93,7 +93,8 @@ do # There should not be a random comma at the end of the webseeds webseeds="${webseeds%,}" - mktorrent -a "${tracker}" "${pkg}" -w "${webseeds}" # "${torrent_location}" + mktorrent -a "${tracker}" "${pkg}" -w "${webseeds}" >/dev/null || + echo "Error making torrent for \"${pkg}\"" fi done done -- cgit v1.1-4-g5e80 From c121ead116f26ed2b36247247fdf4a0d19f063f2 Mon Sep 17 00:00:00 2001 From: Joseph Graham Date: Mon, 16 Jun 2014 14:09:50 +0100 Subject: removed all mirrors but one since pacman2pacman now re-writes the webseeds list to just point to the user's chosen mirror. --- cron-jobs/make_repo_torrents | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents index 797142e..0798a5e 100644 --- a/cron-jobs/make_repo_torrents +++ b/cron-jobs/make_repo_torrents @@ -38,7 +38,11 @@ public_location='/srv/http/repo/public/' tracker='http://t67.eu:6969/announce' # t67.eu is run by Xylon # All mirrors go here -declare -a array=('http://repo.parabolagnulinux.org/' 'https://parabola.goodgnus.com.ar/' 'http://mirror.yandex.ru/mirrors/parabola/' 'http://alfplayer.com/parabola/' 'http://mirror.parlementum.net/') +#declare -a array=('http://repo.parabolagnulinux.org/' 'https://parabola.goodgnus.com.ar/' 'http://mirror.yandex.ru/mirrors/parabola/' 'http://alfplayer.com/parabola/' 'http://mirror.parlementum.net/') + +# I'm removing all mirrors but one since pacman2pacman now re-writes +# the webseeds list to just point to the user's chosen mirror +declare -a array=('http://repo.parabolagnulinux.org/') # I got this function from http://mywiki.wooledge.org/BashFAQ/026 . It # shuffles an array. Uses a global array variable. Must be compact -- cgit v1.1-4-g5e80 From 8e670c23359134af39b1c97173895c09948c2acb Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 24 Jun 2014 22:51:14 -0400 Subject: db-update: get the list of repos the same way as upstream Arch The largest advantage of this is that it implicitly ignores directories we don't want to consider repos. --- config | 2 +- db-update | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/config b/config index 75e88cb..f4c1e15 100644 --- a/config +++ b/config @@ -47,7 +47,7 @@ OURARCHES=(mips64el) ARCHES=(${ARCHARCHES[@]} ${OURARCHES[@]}) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" -PKGEXT=".pkg.tar.xz" +PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" MAKEPKGCONF="~/.makepkg.conf" diff --git a/db-update b/db-update index 4d2cace..a0a3e39 100755 --- a/db-update +++ b/db-update @@ -11,11 +11,18 @@ if [ $# -ge 1 ]; then fi # Find repos with packages to release -repos=($(find "${STAGING}" -mindepth 1 -maxdepth 1 -type d ! -empty -printf '%f ' 2>/dev/null)) +staging_repos=($(find "${STAGING}" -mindepth 1 -type f -name "*${PKGEXT}" -printf '%h\n' | sort -u)) if [ $? -ge 1 ]; then die "Could not read %s" "${STAGING}" fi +repos=() +for staging_repo in "${staging_repos[@]##*/}"; do + if in_array "${staging_repo}" "${PKGREPOS[@]}"; then + repos+=("${staging_repo}") + fi +done + # TODO: this might lock too much (architectures) for repo in "${repos[@]}"; do for pkgarch in "${ARCHES[@]}"; do -- cgit v1.1-4-g5e80 From e029bd5f834a5973ba3be45fd06eed911f26477b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Wed, 25 Jun 2014 02:23:04 -0300 Subject: revert changes on config * fix package extension on PKGEXT variable, because it's generating a bug (e.g. "ERROR Package... already exists in another repository") on all the packages that are being uploaded to the staging folder. --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index f4c1e15..75e88cb 100644 --- a/config +++ b/config @@ -47,7 +47,7 @@ OURARCHES=(mips64el) ARCHES=(${ARCHARCHES[@]} ${OURARCHES[@]}) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" -PKGEXT=".pkg.tar.?z" +PKGEXT=".pkg.tar.xz" SRCEXT=".src.tar.gz" MAKEPKGCONF="~/.makepkg.conf" -- cgit v1.1-4-g5e80 From ee08c7abc25647e8ad038df03ac35144401b6091 Mon Sep 17 00:00:00 2001 From: Joseph Graham Date: Sun, 29 Jun 2014 15:34:37 +0100 Subject: Added an additional tracker. --- cron-jobs/make_repo_torrents | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents index 0798a5e..b5c6749 100644 --- a/cron-jobs/make_repo_torrents +++ b/cron-jobs/make_repo_torrents @@ -34,8 +34,8 @@ esac torrent_location='/srv/http/repo/public/torrents/' public_location='/srv/http/repo/public/' -# Tracker announce URL -tracker='http://t67.eu:6969/announce' # t67.eu is run by Xylon +# Tracker announce URLs, comma seperated +tracker='http://t67.eu:6969/announce,http://tracker.hackcoop.com.ar/announce' # t67.eu is run by Xylon # All mirrors go here #declare -a array=('http://repo.parabolagnulinux.org/' 'https://parabola.goodgnus.com.ar/' 'http://mirror.yandex.ru/mirrors/parabola/' 'http://alfplayer.com/parabola/' 'http://mirror.parlementum.net/') -- cgit v1.1-4-g5e80 From b85f896c2cb614ac22b11c14bd9e7994773309eb Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 4 Jul 2014 22:51:46 -0400 Subject: Fix having a flexible PKGEXT --- config | 2 +- db-update | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/config b/config index 75e88cb..f4c1e15 100644 --- a/config +++ b/config @@ -47,7 +47,7 @@ OURARCHES=(mips64el) ARCHES=(${ARCHARCHES[@]} ${OURARCHES[@]}) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" -PKGEXT=".pkg.tar.xz" +PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" MAKEPKGCONF="~/.makepkg.conf" diff --git a/db-update b/db-update index a0a3e39..3c06f63 100755 --- a/db-update +++ b/db-update @@ -3,7 +3,10 @@ . "$(dirname "$(readlink -e "$0")")/config" . "$(dirname "$(readlink -e "$0")")/db-functions" -shopt -s nullglob +if [[ $STAGING = *luke* ]]; then + set -x + PKGEXT='.pkg.tar.?z' +fi if [ $# -ge 1 ]; then warning "Calling %s with a specific repository is no longer supported" "${0##*/}" @@ -32,11 +35,11 @@ done # check if packages are valid for repo in "${repos[@]}"; do + if ! check_repo_permission "${repo}"; then + die "You don't have permission to update packages in %s" "${repo}" + fi pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXT})) if [ $? -eq 0 ]; then - if [ ${#pkgs[@]} -gt 0 ] && ! check_repo_permission "${repo}"; then - die "You don't have permission to update packages in %s" "${repo}" - fi for pkg in "${pkgs[@]}"; do if [ -h "${pkg}" ]; then die "Package %s is a symbolic link" "${repo}/${pkg##*/}" @@ -95,10 +98,10 @@ done cd "${STAGING}" while read -r file; do pub="${FTP_BASE}/${file}" - if [[ -f $pub ]]; then + if [[ -f "$pub" ]]; then warning "file already exists: %s" "${file}" else mkdir -p -- "${pub%/*}" mv -vn "$file" "$pub" fi -done < <(find other sources -type f) +done < <(find other sources -type f 2>/dev/null) -- cgit v1.1-4-g5e80 From 5fad74e56b72810c9d7f86956236ff45654fb684 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 26 Jul 2014 20:20:33 -0400 Subject: make_repo_torrents: load the location from the common config file --- cron-jobs/make_repo_torrents | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) mode change 100644 => 100755 cron-jobs/make_repo_torrents diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents old mode 100644 new mode 100755 index b5c6749..faee191 --- a/cron-jobs/make_repo_torrents +++ b/cron-jobs/make_repo_torrents @@ -31,8 +31,9 @@ esac # pacman doesn't support multiple different packages of the same name, # so it's OK to just stuff all the torrents into a single directory. -torrent_location='/srv/http/repo/public/torrents/' -public_location='/srv/http/repo/public/' +. "$(dirname "$(readlink -e "$0")")/../config" +public_location="$FTP_BASE/" +torrent_location="$FTP_BASE/torrents/" # Tracker announce URLs, comma seperated tracker='http://t67.eu:6969/announce,http://tracker.hackcoop.com.ar/announce' # t67.eu is run by Xylon -- cgit v1.1-4-g5e80 From 52951d35c69766d0d196fc4cf59fb4864f46d334 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 26 Jul 2014 20:21:14 -0400 Subject: config: set FTP_BASE=~repo/public/main instead of ~repo/public This is to do fancy union stuff to make the web server happy --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index f4c1e15..51d044e 100644 --- a/config +++ b/config @@ -1,5 +1,5 @@ #!/bin/bash # as a hint to text editors -FTP_BASE="/srv/http/repo/public" +FTP_BASE="/srv/http/repo/public/main" SVNREPO="/var/abs" # Repos from Arch -- cgit v1.1-4-g5e80 From b2a763cee1f67751065c39800a59c431ec872bfa Mon Sep 17 00:00:00 2001 From: Joseph Graham Date: Sun, 27 Jul 2014 11:55:01 +0100 Subject: Dividing make_repo_torrents into two scripts, one to make the actual torrent and a wrapper to find any updated packages and call the first script for each... --- cron-jobs/make_repo_torrents | 57 ++++++-------------------------------------- 1 file changed, 7 insertions(+), 50 deletions(-) diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents index faee191..c94d073 100755 --- a/cron-jobs/make_repo_torrents +++ b/cron-jobs/make_repo_torrents @@ -14,7 +14,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# This script depends on `mktorrent' +# This script finds any updated packages and makes calls +# `make_indivudual_torrent' for each of them. username=$( id -un ) @@ -35,71 +36,27 @@ esac public_location="$FTP_BASE/" torrent_location="$FTP_BASE/torrents/" -# Tracker announce URLs, comma seperated -tracker='http://t67.eu:6969/announce,http://tracker.hackcoop.com.ar/announce' # t67.eu is run by Xylon - -# All mirrors go here -#declare -a array=('http://repo.parabolagnulinux.org/' 'https://parabola.goodgnus.com.ar/' 'http://mirror.yandex.ru/mirrors/parabola/' 'http://alfplayer.com/parabola/' 'http://mirror.parlementum.net/') - -# I'm removing all mirrors but one since pacman2pacman now re-writes -# the webseeds list to just point to the user's chosen mirror -declare -a array=('http://repo.parabolagnulinux.org/') - -# I got this function from http://mywiki.wooledge.org/BashFAQ/026 . It -# shuffles an array. Uses a global array variable. Must be compact -# (not a sparse array). The array must be called `array'. -shuffle() { - local i tmp size max rand - - # $RANDOM % (i+1) is biased because of the limited range of $RANDOM - # Compensate by using a range which is a multiple of the array size. - size=${#array[*]} - max=$(( 32768 / size * size )) - - for ((i=size-1; i>0; i--)); do - while (( (rand=$RANDOM) >= max )); do :; done - rand=$(( rand % (i+1) )) - tmp=${array[i]} array[i]=${array[rand]} array[rand]=$tmp - done -} - cd "${torrent_location}" +# Find any directories that might have packages in then find "${public_location}" -name 'os' -type 'd' | while read dir do + # Find any packages find "${dir}" -name '*\.pkg\.tar\.xz' | while read pkg do - pkg_name="${pkg##*/}" - if [[ -h "${pkg}" ]] # check if it's a symbolic link then # We get the target of the symlink pkg=$( readlink -f "${pkg}" ) fi + # If a .torrent file does not already exist for this package, we call + # `make_individual_torrent' to make it. if ! [[ -f "${torrent_location}${pkg_name}.torrent" ]] then - # We need to make a comma seperated list of webseeds (this is passed - # as a single argument to mktorrent) - webseeds='' - - # Randomize the order of the list of webseeds because I - # don't know if transmission might always use the one at - # the top otherwize. - shuffle - - for prefix in "${array[@]}" - do - webseeds+="${prefix}${pkg#${public_location}}," - done - - # There should not be a random comma at the end of the webseeds - webseeds="${webseeds%,}" - - mktorrent -a "${tracker}" "${pkg}" -w "${webseeds}" >/dev/null || - echo "Error making torrent for \"${pkg}\"" + /srv/http/repo/dbscripts/make_individual_torrent "${pkg}" fi done done -- cgit v1.1-4-g5e80 From db2ea03e53a3aed1c8350811be70a31c7ea0ef53 Mon Sep 17 00:00:00 2001 From: Joseph Graham Date: Sun, 27 Jul 2014 11:55:08 +0100 Subject: Added new file --- make_individual_torrent | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 make_individual_torrent diff --git a/make_individual_torrent b/make_individual_torrent new file mode 100644 index 0000000..0cfee56 --- /dev/null +++ b/make_individual_torrent @@ -0,0 +1,44 @@ +#! /bin/bash +# Copyright (C) 2014 Joseph Graham +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# This script is called by `make_repo_torrents' to make a torrent. It +# depends on `mktorrent'. It takes the following arg: +# $1 - path of package + +# Comma seperated list of trackers, no spaces +# t67.eu is run by Xylon, hackcoop by fauno & friends +trackers='http://t67.eu:6969/announce,http://tracker.hackcoop.com.ar/announce' + +# This mirror is put as a webseed. Which mirror we use for a webseed +# doesn't really matter since it's re-written on the client machine by +# pacman2pacman so it won't normally be used anyway. +seed_url='http://repo.parabolagnulinux.org/' + +if [[ -z "${1}" ]] +then + echo "Error. First arg must be the path of the package." + echo 1 +fi + +pkg="${1}" + +pkg_name="${pkg##*/}" + +# URL of the actual package for the webseed +webseed="${seed_url}${pkg#${public_location}}" + +mktorrent -a "${trackers}" "${pkg}" -w "${webseed}" >/dev/null || +echo "Error making torrent for \"${pkg}\"" -- cgit v1.1-4-g5e80 From 6790ac43f2f79fea6adcfef2a10d3c4105574216 Mon Sep 17 00:00:00 2001 From: Joseph Graham Date: Sun, 27 Jul 2014 12:07:06 +0100 Subject: fixed loads of mistakes I make --- cron-jobs/make_repo_torrents | 4 +++- make_individual_torrent | 10 +++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents index c94d073..a5d72a5 100755 --- a/cron-jobs/make_repo_torrents +++ b/cron-jobs/make_repo_torrents @@ -46,6 +46,8 @@ do find "${dir}" -name '*\.pkg\.tar\.xz' | while read pkg do + pkg_name="${pkg##*/}" + if [[ -h "${pkg}" ]] # check if it's a symbolic link then # We get the target of the symlink @@ -56,7 +58,7 @@ do # `make_individual_torrent' to make it. if ! [[ -f "${torrent_location}${pkg_name}.torrent" ]] then - /srv/http/repo/dbscripts/make_individual_torrent "${pkg}" + /srv/http/repo/dbscripts/make_individual_torrent "${pkg}" "${public_location}" fi done done diff --git a/make_individual_torrent b/make_individual_torrent index 0cfee56..e5b7d8c 100644 --- a/make_individual_torrent +++ b/make_individual_torrent @@ -15,8 +15,9 @@ # along with this program. If not, see . # This script is called by `make_repo_torrents' to make a torrent. It -# depends on `mktorrent'. It takes the following arg: +# depends on `mktorrent'. It takes the following args: # $1 - path of package +# $2 - public location # Comma seperated list of trackers, no spaces # t67.eu is run by Xylon, hackcoop by fauno & friends @@ -33,7 +34,14 @@ then echo 1 fi +if [[ -z "${2}" ]] +then + echo "Error. Second arg must be the public location." + echo 1 +fi + pkg="${1}" +public_location="${2}" pkg_name="${pkg##*/}" -- cgit v1.1-4-g5e80 From 01f85e9950540e6718e140882051dd8f3a9ba9a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?coadde=20=5BM=C3=A1rcio=20Alexandre=20Silva=20Delgado=5D?= Date: Wed, 3 Sep 2014 17:02:14 -0300 Subject: add nonprism-testing repo --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 51d044e..92c059c 100644 --- a/config +++ b/config @@ -9,7 +9,7 @@ OURREPOS=('libre' 'libre-testing' 'libre-multilib' 'libre-multilib-testing') # User repos USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan') # Community project repos -PROJREPOS=('pcr' 'kernels' 'cross' 'java' 'nonprism') +PROJREPOS=('nonprism' 'nonprism-testing' 'pcr' 'kernels' 'cross' 'java') # Remote repos PKGREPOS=("${ARCHREPOS[@]}" "${OURREPOS[@]}" "${USERREPOS[@]}" "${PROJREPOS[@]}") PKGPOOL='pool/parabola' -- cgit v1.1-4-g5e80 From a264b078cde98c6ad3766d1bdf69ca331f01baa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Mon, 29 Sep 2014 02:22:28 -0300 Subject: config: set FTP_BASE=/srv/repo/main instead of /srv/http/repo/public/main --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 92c059c..544a8e9 100644 --- a/config +++ b/config @@ -1,5 +1,5 @@ #!/bin/bash # as a hint to text editors -FTP_BASE="/srv/http/repo/public/main" +FTP_BASE="/srv/repo/main" SVNREPO="/var/abs" # Repos from Arch -- cgit v1.1-4-g5e80 From 5e9c23a6cc430138358c97544b1a9219d643b6a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Mas=C5=82owski?= Date: Tue, 30 Sep 2014 21:18:41 +0200 Subject: Fix make_repo_torrents for the new server. --- cron-jobs/make_repo_torrents | 3 ++- make_individual_torrent | 0 2 files changed, 2 insertions(+), 1 deletion(-) mode change 100644 => 100755 make_individual_torrent diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents index a5d72a5..fec4698 100755 --- a/cron-jobs/make_repo_torrents +++ b/cron-jobs/make_repo_torrents @@ -32,6 +32,7 @@ esac # pacman doesn't support multiple different packages of the same name, # so it's OK to just stuff all the torrents into a single directory. +script_directory="$(dirname "$(readlink -e "$0")")/.." . "$(dirname "$(readlink -e "$0")")/../config" public_location="$FTP_BASE/" torrent_location="$FTP_BASE/torrents/" @@ -58,7 +59,7 @@ do # `make_individual_torrent' to make it. if ! [[ -f "${torrent_location}${pkg_name}.torrent" ]] then - /srv/http/repo/dbscripts/make_individual_torrent "${pkg}" "${public_location}" + "$script_directory/make_individual_torrent" "${pkg}" "${public_location}" fi done done diff --git a/make_individual_torrent b/make_individual_torrent old mode 100644 new mode 100755 -- cgit v1.1-4-g5e80 From 724674000a5c4ac1ce87db0b5f86b51d1d070a41 Mon Sep 17 00:00:00 2001 From: Joseph Graham Date: Fri, 3 Oct 2014 22:21:44 +0100 Subject: Fixed typo in comments --- cron-jobs/make_repo_torrents | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents index fec4698..fc723f1 100755 --- a/cron-jobs/make_repo_torrents +++ b/cron-jobs/make_repo_torrents @@ -14,7 +14,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# This script finds any updated packages and makes calls +# This script finds any updated packages and calls # `make_indivudual_torrent' for each of them. username=$( id -un ) -- cgit v1.1-4-g5e80 From 2c72fef7bd097105e57e05a4a49d0eda060735ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Mas=C5=82owski?= Date: Sat, 4 Oct 2014 21:21:49 +0200 Subject: Update lastsync after db-sync or cleanup. --- db-functions | 1 + db-sync | 2 ++ 2 files changed, 3 insertions(+) diff --git a/db-functions b/db-functions index 2eeffbb..dd8e2dc 100644 --- a/db-functions +++ b/db-functions @@ -141,6 +141,7 @@ cleanup() { if (( REPO_MODIFIED )); then date +%s > "${FTP_BASE}/lastupdate" + date -u +%s > "${FTP_BASE}/lastsync" fi [ "$1" ] && exit "$1" diff --git a/db-sync b/db-sync index 58e211d..35b6489 100755 --- a/db-sync +++ b/db-sync @@ -174,6 +174,8 @@ init() { "${FTP_BASE}/${srcpool}/" done + date -u +%s > "${FTP_BASE}/lastsync" + # Cleanup unset blacklist whitelists _arch _repo repo_file } -- cgit v1.1-4-g5e80 From 548dad674dd2e2e59402981522676284eee6cee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Mas=C5=82owski?= Date: Sun, 26 Oct 2014 20:22:07 +0100 Subject: db-list-unsigned-packages.py: support listing keys that signed the packages. --- db-list-unsigned-packages.py | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/db-list-unsigned-packages.py b/db-list-unsigned-packages.py index 36be93a..80cff51 100755 --- a/db-list-unsigned-packages.py +++ b/db-list-unsigned-packages.py @@ -21,23 +21,35 @@ unsigned packages in the database at standard input of repo named in the first argument and specified for architectures listed in the following arguments (usually the one of the database or any, default is to list all). + +If the --keyset argument is passed, print the key fingerprint of every +signed package. """ +import base64 +import subprocess import sys import tarfile def main(): """Do the job.""" + check_keys = False + if "--keyset" in sys.argv: + sys.argv.remove("--keyset") + check_keys = True repo = sys.argv[1] pkgarches = frozenset(name.encode("utf-8") for name in sys.argv[2:]) + packages = [] + keys = [] with tarfile.open(fileobj=sys.stdin.buffer) as archive: for entry in archive: if entry.name.endswith("/desc"): content = archive.extractfile(entry) skip = False is_arch = False + key = None for line in content: if is_arch: is_arch = False @@ -46,12 +58,38 @@ def main(): break if line == b"%PGPSIG%\n": skip = True # signed - break + key = b"" + if check_keys: + continue + else: + break if line == b"%ARCH%\n": is_arch = True + continue + if key is not None: + if line.strip(): + key += line.strip() + else: + break + if check_keys and key: + key_binary = base64.b64decode(key) + keys.append(key_binary) + packages.append(repo + "/" + entry.name[:-5]) if skip: continue print(repo + "/" + entry.name[:-5]) + if check_keys and keys: + # We have collected all signed package names in packages and + # all keys in keys. Let's now ask gpg to list all signatures + # and find which keys made them. + packets = subprocess.check_output(("gpg", "--list-packets"), + input=b"".join(keys)) + i = 0 + for line in packets.decode("latin1").split("\n"): + if line.startswith(":signature packet:"): + keyid = line[line.index("keyid ") + len("keyid "):] + print(packages[i], keyid) + i += 1 if __name__ == "__main__": -- cgit v1.1-4-g5e80 From dd63290fd4c15b6220ade6ef4d6f8f0e4878944e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Fabian=20Silva=20Delgado?= Date: Wed, 15 Apr 2015 11:40:23 -0300 Subject: add nonsystemd and nonsystemd-testing repos --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 544a8e9..fdf3ef9 100644 --- a/config +++ b/config @@ -9,7 +9,7 @@ OURREPOS=('libre' 'libre-testing' 'libre-multilib' 'libre-multilib-testing') # User repos USERREPOS=('~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' '~jorginho' '~coadde' '~drtan') # Community project repos -PROJREPOS=('nonprism' 'nonprism-testing' 'pcr' 'kernels' 'cross' 'java') +PROJREPOS=('nonsystemd' 'nonsystemd-testing' 'nonprism' 'nonprism-testing' 'pcr' 'kernels' 'cross' 'java') # Remote repos PKGREPOS=("${ARCHREPOS[@]}" "${OURREPOS[@]}" "${USERREPOS[@]}" "${PROJREPOS[@]}") PKGPOOL='pool/parabola' -- cgit v1.1-4-g5e80