From 3c1eb5b59792fbaafde12a7408201fb1e0a77df6 Mon Sep 17 00:00:00 2001
From: Aaron Griffin <aaronmgriffin@gmail.com>
Date: Thu, 29 Oct 2009 19:10:26 -0700
Subject: Rename createFileLists to be more like the others

Signed-off-by: Aaron Griffin <aaronmgriffin@gmail.com>
---
 cron-jobs/create-filelists | 82 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 82 insertions(+)
 create mode 100755 cron-jobs/create-filelists

(limited to 'cron-jobs/create-filelists')

diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
new file mode 100755
index 0000000..d936853
--- /dev/null
+++ b/cron-jobs/create-filelists
@@ -0,0 +1,82 @@
+#!/bin/bash
+
+reposdir=/srv/ftp/
+targetdir=$reposdir
+repos="core extra testing community"
+
+. "$(dirname $0)/../db-functions"
+. "$(dirname $0)/../config"
+
+if [ -f "/tmp/createFileList.lock" ]; then
+    echo "Error: createFileList allready in progress."
+    exit 1
+fi
+
+touch "/tmp/createFileList.lock" || exit 1
+TMPDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1
+CACHEDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1
+
+#adjust the nice level to run at a lower priority
+/usr/bin/renice +10 -p $$ > /dev/null
+
+case "${DBEXT}" in
+	*.gz)  TAR_OPT="z" ;;
+	*.bz2) TAR_OPT="j" ;;
+	*.xz)  TAR_OPT="J" ;;
+	*) echo "Unknown compression type for DBEXT=${DBEXT}" && exit 1 ;;
+esac
+
+FILESEXT="${DBEXT//db/files}"
+
+cd $reposdir
+for repo in $repos; do
+    REPO_DB_FILE=${repo}$FILESEXT
+    for arch in ${ARCHES[@]}; do
+        repodir=${repo}/os/${arch}/
+        cached="no"
+
+        # extract old file archive
+        if [ -f ${targetdir}${repodir}${REPO_DB_FILE} ]; then
+            mkdir -p ${CACHEDIR}/${repodir}
+            bsdtar -xf ${targetdir}${repodir}${REPO_DB_FILE} -C ${CACHEDIR}/${repodir}
+            cached="yes"
+        fi
+
+        # create file lists
+        for pkg in $repodir*${PKGEXT}; do
+            basename=$(basename $pkg)
+            pkgname=$(getpkgname $basename)
+            tmppkgdir=${TMPDIR}/${repodir}${pkgname}/
+            mkdir -p $tmppkgdir
+            if [ -f "${CACHEDIR}/${repodir}${pkgname}/files" ]; then
+#               echo "cache: $pkgname"
+                mv ${CACHEDIR}/${repodir}${pkgname}/files ${tmppkgdir}files
+            else
+#               echo "$repo/$arch: $pkgname"
+                echo '%FILES%' > ${tmppkgdir}files
+                bsdtar --exclude=.* -tf $pkg >> ${tmppkgdir}files
+                cached="no"
+            fi
+        done
+
+        # create new file archive
+        if [ "$cached" == "no" ]; then
+            # at least one package has changed, so let's rebuild the archive
+#           echo "creating ${REPO_DB_FILE}/${arch}"
+            pkgdir=${targetdir}${repodir}
+            mkdir -p $pkgdir
+            cd ${TMPDIR}/${repodir}
+            [ -f "${pkgdir}${REPO_DB_FILE}.old" ] && rm "${pkgdir}${REPO_DB_FILE}.old"
+            [ -f "${pkgdir}${REPO_DB_FILE}" ] && mv "${pkgdir}${REPO_DB_FILE}" "${pkgdir}${REPO_DB_FILE}.old"
+            bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f ${pkgdir}${REPO_DB_FILE} *
+        fi
+
+        cd $reposdir
+    done
+done
+
+cd - >/dev/null
+rm -rf $TMPDIR || exit 1
+rm -rf $CACHEDIR || exit 1
+rm -f "/tmp/createFileList.lock" || exit 1
+# echo 'done'
-- 
cgit v1.2.3-2-g168b


From 38c7241ad2528b29a65a0b81b980e76d1046e94e Mon Sep 17 00:00:00 2001
From: Aaron Griffin <aaronmgriffin@gmail.com>
Date: Thu, 29 Oct 2009 19:18:41 -0700
Subject: Cleanup create-filelists

* Quote absolutely everything
* Move directory slashes around, for readability
* Move a cd to the top of the loop

Signed-off-by: Aaron Griffin <aaronmgriffin@gmail.com>
---
 cron-jobs/create-filelists | 55 +++++++++++++++++++++++-----------------------
 1 file changed, 27 insertions(+), 28 deletions(-)

(limited to 'cron-jobs/create-filelists')

diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
index d936853..b49f722 100755
--- a/cron-jobs/create-filelists
+++ b/cron-jobs/create-filelists
@@ -1,7 +1,7 @@
 #!/bin/bash
 
-reposdir=/srv/ftp/
-targetdir=$reposdir
+reposdir="/srv/ftp"
+targetdir="/srv/ftp"
 repos="core extra testing community"
 
 . "$(dirname $0)/../db-functions"
@@ -13,8 +13,8 @@ if [ -f "/tmp/createFileList.lock" ]; then
 fi
 
 touch "/tmp/createFileList.lock" || exit 1
-TMPDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1
-CACHEDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1
+TMPDIR="$(mktemp -d /tmp/createFileList.XXXXXX)" || exit 1
+CACHEDIR="$(mktemp -d /tmp/createFileList.XXXXXX)" || exit 1
 
 #adjust the nice level to run at a lower priority
 /usr/bin/renice +10 -p $$ > /dev/null
@@ -28,33 +28,34 @@ esac
 
 FILESEXT="${DBEXT//db/files}"
 
-cd $reposdir
 for repo in $repos; do
-    REPO_DB_FILE=${repo}$FILESEXT
+    REPO_DB_FILE="${repo}$FILESEXT"
     for arch in ${ARCHES[@]}; do
-        repodir=${repo}/os/${arch}/
+        cd "$reposdir"
+
+        repodir="${repo}/os/${arch}"
         cached="no"
 
         # extract old file archive
-        if [ -f ${targetdir}${repodir}${REPO_DB_FILE} ]; then
-            mkdir -p ${CACHEDIR}/${repodir}
-            bsdtar -xf ${targetdir}${repodir}${REPO_DB_FILE} -C ${CACHEDIR}/${repodir}
+        if [ -f "${targetdir}/${repodir}/${REPO_DB_FILE}" ]; then
+            mkdir -p "${CACHEDIR}/${repodir}"
+            bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${CACHEDIR}/${repodir}"
             cached="yes"
         fi
 
         # create file lists
         for pkg in $repodir*${PKGEXT}; do
-            basename=$(basename $pkg)
-            pkgname=$(getpkgname $basename)
-            tmppkgdir=${TMPDIR}/${repodir}${pkgname}/
-            mkdir -p $tmppkgdir
-            if [ -f "${CACHEDIR}/${repodir}${pkgname}/files" ]; then
+            basename="$(basename "$pkg")"
+            pkgname="$(getpkgname "$basename")"
+            tmppkgdir="${TMPDIR}/${repodir}/${pkgname}"
+            mkdir -p "$tmppkgdir"
+            if [ -f "${CACHEDIR}/${repodir}/${pkgname}/files" ]; then
 #               echo "cache: $pkgname"
-                mv ${CACHEDIR}/${repodir}${pkgname}/files ${tmppkgdir}files
+                mv "${CACHEDIR}/${repodir}$/{pkgname}/files" "${tmppkgdir}/files"
             else
 #               echo "$repo/$arch: $pkgname"
-                echo '%FILES%' > ${tmppkgdir}files
-                bsdtar --exclude=.* -tf $pkg >> ${tmppkgdir}files
+                echo '%FILES%' > "${tmppkgdir}/files"
+                bsdtar --exclude=.* -tf "$pkg" >> "${tmppkgdir}/files"
                 cached="no"
             fi
         done
@@ -63,20 +64,18 @@ for repo in $repos; do
         if [ "$cached" == "no" ]; then
             # at least one package has changed, so let's rebuild the archive
 #           echo "creating ${REPO_DB_FILE}/${arch}"
-            pkgdir=${targetdir}${repodir}
-            mkdir -p $pkgdir
-            cd ${TMPDIR}/${repodir}
-            [ -f "${pkgdir}${REPO_DB_FILE}.old" ] && rm "${pkgdir}${REPO_DB_FILE}.old"
-            [ -f "${pkgdir}${REPO_DB_FILE}" ] && mv "${pkgdir}${REPO_DB_FILE}" "${pkgdir}${REPO_DB_FILE}.old"
-            bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f ${pkgdir}${REPO_DB_FILE} *
+            pkgdir="${targetdir}/${repodir}"
+            mkdir -p "$pkgdir"
+            cd "${TMPDIR}/${repodir}"
+            [ -f "${pkgdir}/${REPO_DB_FILE}.old" ] && rm "${pkgdir}/${REPO_DB_FILE}.old"
+            [ -f "${pkgdir}/${REPO_DB_FILE}" ] && mv "${pkgdir}/${REPO_DB_FILE}" "${pkgdir}/${REPO_DB_FILE}.old"
+            bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f "${pkgdir}/${REPO_DB_FILE}" *
         fi
-
-        cd $reposdir
     done
 done
 
 cd - >/dev/null
-rm -rf $TMPDIR || exit 1
-rm -rf $CACHEDIR || exit 1
+rm -rf "$TMPDIR" || exit 1
+rm -rf "$CACHEDIR" || exit 1
 rm -f "/tmp/createFileList.lock" || exit 1
 # echo 'done'
-- 
cgit v1.2.3-2-g168b


From 78c6a7916836fd10fb232d617b6eca3ae5c48089 Mon Sep 17 00:00:00 2001
From: Aaron Griffin <aaronmgriffin@gmail.com>
Date: Thu, 29 Oct 2009 19:19:44 -0700
Subject: create-filelists: Call getpkgname for real

Signed-off-by: Aaron Griffin <aaronmgriffin@gmail.com>
---
 cron-jobs/create-filelists | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

(limited to 'cron-jobs/create-filelists')

diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
index b49f722..35325cf 100755
--- a/cron-jobs/create-filelists
+++ b/cron-jobs/create-filelists
@@ -45,8 +45,7 @@ for repo in $repos; do
 
         # create file lists
         for pkg in $repodir*${PKGEXT}; do
-            basename="$(basename "$pkg")"
-            pkgname="$(getpkgname "$basename")"
+            pkgname="$(getpkgname "$pkg")"
             tmppkgdir="${TMPDIR}/${repodir}/${pkgname}"
             mkdir -p "$tmppkgdir"
             if [ -f "${CACHEDIR}/${repodir}/${pkgname}/files" ]; then
-- 
cgit v1.2.3-2-g168b


From 1857fca2d204011852cf2309b66381b2fb4f6535 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20B=C3=A4chler?= <thomas@archlinux.org>
Date: Sat, 31 Oct 2009 19:06:33 +0100
Subject: Fix a typo introduced in 38c7241ad2528b29a65a0b81b980e76d1046e94e

---
 cron-jobs/create-filelists | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'cron-jobs/create-filelists')

diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
index 35325cf..727fe58 100755
--- a/cron-jobs/create-filelists
+++ b/cron-jobs/create-filelists
@@ -50,7 +50,7 @@ for repo in $repos; do
             mkdir -p "$tmppkgdir"
             if [ -f "${CACHEDIR}/${repodir}/${pkgname}/files" ]; then
 #               echo "cache: $pkgname"
-                mv "${CACHEDIR}/${repodir}$/{pkgname}/files" "${tmppkgdir}/files"
+                mv "${CACHEDIR}/${repodir}/${pkgname}/files" "${tmppkgdir}/files"
             else
 #               echo "$repo/$arch: $pkgname"
                 echo '%FILES%' > "${tmppkgdir}/files"
-- 
cgit v1.2.3-2-g168b


From 782e8d67158115e8040215bcfef7e8ba54c6d04e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20B=C3=A4chler?= <thomas@archlinux.org>
Date: Sat, 31 Oct 2009 19:17:39 +0100
Subject: create-filelists: add community-testing

---
 cron-jobs/create-filelists | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'cron-jobs/create-filelists')

diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
index 727fe58..840535e 100755
--- a/cron-jobs/create-filelists
+++ b/cron-jobs/create-filelists
@@ -2,7 +2,7 @@
 
 reposdir="/srv/ftp"
 targetdir="/srv/ftp"
-repos="core extra testing community"
+repos="core extra testing community community-testing"
 
 . "$(dirname $0)/../db-functions"
 . "$(dirname $0)/../config"
-- 
cgit v1.2.3-2-g168b


From c54f6c91dca7d84480f22aee60dda5861c73712e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20B=C3=A4chler?= <thomas@archlinux.org>
Date: Sat, 31 Oct 2009 19:28:38 +0100
Subject: create-filelists: fix another typo

---
 cron-jobs/create-filelists | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'cron-jobs/create-filelists')

diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
index 840535e..2a15e6a 100755
--- a/cron-jobs/create-filelists
+++ b/cron-jobs/create-filelists
@@ -44,7 +44,7 @@ for repo in $repos; do
         fi
 
         # create file lists
-        for pkg in $repodir*${PKGEXT}; do
+        for pkg in $repodir/*${PKGEXT}; do
             pkgname="$(getpkgname "$pkg")"
             tmppkgdir="${TMPDIR}/${repodir}/${pkgname}"
             mkdir -p "$tmppkgdir"
-- 
cgit v1.2.3-2-g168b


From 27e8f1071c18b2f0cbf4deb84a1222db64a3d5dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20B=C3=A4chler?= <thomas@archlinux.org>
Date: Sat, 31 Oct 2009 20:07:17 +0100
Subject: create-filelists: Add pkgver-pkgrel to directory name

In the past, pkgname-pkgver-pkgrel was extracted from the filename and sadly named "pkgname". Restore this behaviour
---
 cron-jobs/create-filelists | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

(limited to 'cron-jobs/create-filelists')

diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
index 2a15e6a..c9d7db9 100755
--- a/cron-jobs/create-filelists
+++ b/cron-jobs/create-filelists
@@ -46,11 +46,12 @@ for repo in $repos; do
         # create file lists
         for pkg in $repodir/*${PKGEXT}; do
             pkgname="$(getpkgname "$pkg")"
-            tmppkgdir="${TMPDIR}/${repodir}/${pkgname}"
+            pkgver="$(getpkgver "$pkg")"
+            tmppkgdir="${TMPDIR}/${repodir}/${pkgname}-${pkgver}"
             mkdir -p "$tmppkgdir"
-            if [ -f "${CACHEDIR}/${repodir}/${pkgname}/files" ]; then
+            if [ -f "${CACHEDIR}/${repodir}/${pkgname}-${pkgver}/files" ]; then
 #               echo "cache: $pkgname"
-                mv "${CACHEDIR}/${repodir}/${pkgname}/files" "${tmppkgdir}/files"
+                mv "${CACHEDIR}/${repodir}/${pkgname}-${pkgver}/files" "${tmppkgdir}/files"
             else
 #               echo "$repo/$arch: $pkgname"
                 echo '%FILES%' > "${tmppkgdir}/files"
-- 
cgit v1.2.3-2-g168b


From 1d6421c7a12e9a07d00ec4de999ae95f7dd0874e Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Sat, 27 Feb 2010 10:47:52 -0600
Subject: create-filelists: general cleanups

* Specify lock name once
* Use new script name everywhere
* Clean up tabs/spaces and add a modeline. This isn't necessarily the one we
  wanted to standardize on, but I picked the one the entire file is written
  to at the moment.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 cron-jobs/create-filelists | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

(limited to 'cron-jobs/create-filelists')

diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
index c9d7db9..e90da00 100755
--- a/cron-jobs/create-filelists
+++ b/cron-jobs/create-filelists
@@ -3,27 +3,28 @@
 reposdir="/srv/ftp"
 targetdir="/srv/ftp"
 repos="core extra testing community community-testing"
+lock="/tmp/create-filelists.lock"
 
 . "$(dirname $0)/../db-functions"
 . "$(dirname $0)/../config"
 
-if [ -f "/tmp/createFileList.lock" ]; then
-    echo "Error: createFileList allready in progress."
+if [ -f "$lock" ]; then
+    echo "Error: create-filelists already in progress."
     exit 1
 fi
 
-touch "/tmp/createFileList.lock" || exit 1
-TMPDIR="$(mktemp -d /tmp/createFileList.XXXXXX)" || exit 1
-CACHEDIR="$(mktemp -d /tmp/createFileList.XXXXXX)" || exit 1
+touch "$lock" || exit 1
+TMPDIR="$(mktemp -d /tmp/create-filelists.XXXXXX)" || exit 1
+CACHEDIR="$(mktemp -d /tmp/create-filelists.XXXXXX)" || exit 1
 
 #adjust the nice level to run at a lower priority
 /usr/bin/renice +10 -p $$ > /dev/null
 
 case "${DBEXT}" in
-	*.gz)  TAR_OPT="z" ;;
-	*.bz2) TAR_OPT="j" ;;
-	*.xz)  TAR_OPT="J" ;;
-	*) echo "Unknown compression type for DBEXT=${DBEXT}" && exit 1 ;;
+    *.gz)  TAR_OPT="z" ;;
+    *.bz2) TAR_OPT="j" ;;
+    *.xz)  TAR_OPT="J" ;;
+    *) echo "Unknown compression type for DBEXT=${DBEXT}" && exit 1 ;;
 esac
 
 FILESEXT="${DBEXT//db/files}"
@@ -77,5 +78,7 @@ done
 cd - >/dev/null
 rm -rf "$TMPDIR" || exit 1
 rm -rf "$CACHEDIR" || exit 1
-rm -f "/tmp/createFileList.lock" || exit 1
+rm -f "$lock" || exit 1
 # echo 'done'
+
+# vim: set ts=4 sw=4 et ft=sh:
-- 
cgit v1.2.3-2-g168b


From 6587ebadf6b3a0d8e58be1ab7169a4c2f88ed57c Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Sat, 27 Feb 2010 10:58:22 -0600
Subject: create-filelists: s/REPO_DB_FILE/FILES_DB_FILE/g

This will set up changes soon to come where we actually use the real repos
DB file so I don't want variable name confusion.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 cron-jobs/create-filelists | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

(limited to 'cron-jobs/create-filelists')

diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
index e90da00..a0b6a57 100755
--- a/cron-jobs/create-filelists
+++ b/cron-jobs/create-filelists
@@ -30,7 +30,7 @@ esac
 FILESEXT="${DBEXT//db/files}"
 
 for repo in $repos; do
-    REPO_DB_FILE="${repo}$FILESEXT"
+    FILES_DB_FILE="${repo}$FILESEXT"
     for arch in ${ARCHES[@]}; do
         cd "$reposdir"
 
@@ -38,9 +38,9 @@ for repo in $repos; do
         cached="no"
 
         # extract old file archive
-        if [ -f "${targetdir}/${repodir}/${REPO_DB_FILE}" ]; then
+        if [ -f "${targetdir}/${repodir}/${FILES_DB_FILE}" ]; then
             mkdir -p "${CACHEDIR}/${repodir}"
-            bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${CACHEDIR}/${repodir}"
+            bsdtar -xf "${targetdir}/${repodir}/${FILES_DB_FILE}" -C "${CACHEDIR}/${repodir}"
             cached="yes"
         fi
 
@@ -64,13 +64,13 @@ for repo in $repos; do
         # create new file archive
         if [ "$cached" == "no" ]; then
             # at least one package has changed, so let's rebuild the archive
-#           echo "creating ${REPO_DB_FILE}/${arch}"
+#           echo "creating ${FILES_DB_FILE}/${arch}"
             pkgdir="${targetdir}/${repodir}"
             mkdir -p "$pkgdir"
             cd "${TMPDIR}/${repodir}"
-            [ -f "${pkgdir}/${REPO_DB_FILE}.old" ] && rm "${pkgdir}/${REPO_DB_FILE}.old"
-            [ -f "${pkgdir}/${REPO_DB_FILE}" ] && mv "${pkgdir}/${REPO_DB_FILE}" "${pkgdir}/${REPO_DB_FILE}.old"
-            bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f "${pkgdir}/${REPO_DB_FILE}" *
+            [ -f "${pkgdir}/${FILES_DB_FILE}.old" ] && rm "${pkgdir}/${FILES_DB_FILE}.old"
+            [ -f "${pkgdir}/${FILES_DB_FILE}" ] && mv "${pkgdir}/${FILES_DB_FILE}" "${pkgdir}/${FILES_DB_FILE}.old"
+            bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f "${pkgdir}/${FILES_DB_FILE}" *
         fi
     done
 done
-- 
cgit v1.2.3-2-g168b


From ff1530def072daf95f077ec0f8a4d984da4304d6 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Sat, 27 Feb 2010 11:27:13 -0600
Subject: create-filelists: rework the package loop completely

Instead of wasting time extracting .PKGINFO twice from every single package
in the repos, use the package DB to eliminate most of the heavy lifting.
This way we only need to worry about looking at the packages that actually
have changed since the last time we built the package database.

This should give a noticeable performance increase to this job in addition to
reducing IO load and unnecessary reading of every package file.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 cron-jobs/create-filelists | 41 ++++++++++++++++++++++++++++-------------
 1 file changed, 28 insertions(+), 13 deletions(-)

(limited to 'cron-jobs/create-filelists')

diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
index a0b6a57..6091bf4 100755
--- a/cron-jobs/create-filelists
+++ b/cron-jobs/create-filelists
@@ -14,8 +14,12 @@ if [ -f "$lock" ]; then
 fi
 
 touch "$lock" || exit 1
-TMPDIR="$(mktemp -d /tmp/create-filelists.XXXXXX)" || exit 1
-CACHEDIR="$(mktemp -d /tmp/create-filelists.XXXXXX)" || exit 1
+# location where the package DB is extracted so we know what to include
+DBDIR="$(mktemp -d /tmp/create-filelists.dbdir.XXXXXX)" || exit 1
+# location where the old files DB is extracted to save us some work
+CACHEDIR="$(mktemp -d /tmp/create-filelists.cachedir.XXXXXX)" || exit 1
+# location where the new files DB is built up and eventually zipped
+TMPDIR="$(mktemp -d /tmp/create-filelists.tmpdir.XXXXXX)" || exit 1
 
 #adjust the nice level to run at a lower priority
 /usr/bin/renice +10 -p $$ > /dev/null
@@ -30,33 +34,45 @@ esac
 FILESEXT="${DBEXT//db/files}"
 
 for repo in $repos; do
+    REPO_DB_FILE="${repo}$DBEXT"
     FILES_DB_FILE="${repo}$FILESEXT"
     for arch in ${ARCHES[@]}; do
+#       echo "Running for architecture $arch, repo $repo"
         cd "$reposdir"
 
         repodir="${repo}/os/${arch}"
         cached="no"
 
+        # extract package db archive
+        if [ -f "${targetdir}/${repodir}/${REPO_DB_FILE}" ]; then
+            mkdir -p "${DBDIR}/${repodir}"
+#           echo "extracting $REPO_DB_FILE"
+            bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${DBDIR}/${repodir}"
+        else
+            echo "Fail! Does the repo $repo with arch $arch even exist?"
+            continue
+        fi
+
         # extract old file archive
         if [ -f "${targetdir}/${repodir}/${FILES_DB_FILE}" ]; then
             mkdir -p "${CACHEDIR}/${repodir}"
+#           echo "extracting $FILES_DB_FILE"
             bsdtar -xf "${targetdir}/${repodir}/${FILES_DB_FILE}" -C "${CACHEDIR}/${repodir}"
             cached="yes"
         fi
 
         # create file lists
-        for pkg in $repodir/*${PKGEXT}; do
-            pkgname="$(getpkgname "$pkg")"
-            pkgver="$(getpkgver "$pkg")"
-            tmppkgdir="${TMPDIR}/${repodir}/${pkgname}-${pkgver}"
+        for pkg in $(ls ${DBDIR}/${repodir}); do
+            tmppkgdir="${TMPDIR}/${repodir}/${pkg}"
             mkdir -p "$tmppkgdir"
-            if [ -f "${CACHEDIR}/${repodir}/${pkgname}-${pkgver}/files" ]; then
-#               echo "cache: $pkgname"
-                mv "${CACHEDIR}/${repodir}/${pkgname}-${pkgver}/files" "${tmppkgdir}/files"
+            if [ -f "${CACHEDIR}/${repodir}/${pkg}/files" ]; then
+#               echo "cache: $pkg"
+                mv "${CACHEDIR}/${repodir}/${pkg}/files" "${tmppkgdir}/files"
             else
-#               echo "$repo/$arch: $pkgname"
+#               echo "not cache: $repo/$arch: $pkg"
+                filename=$(grep -A1 '^%FILENAME%$' "${DBDIR}/${repodir}/${pkg}/desc" | tail -n1)
                 echo '%FILES%' > "${tmppkgdir}/files"
-                bsdtar --exclude=.* -tf "$pkg" >> "${tmppkgdir}/files"
+                bsdtar --exclude=.* -tf "$repodir/$filename" >> "${tmppkgdir}/files"
                 cached="no"
             fi
         done
@@ -76,8 +92,7 @@ for repo in $repos; do
 done
 
 cd - >/dev/null
-rm -rf "$TMPDIR" || exit 1
-rm -rf "$CACHEDIR" || exit 1
+rm -rf "$TMPDIR" "$CACHEDIR" "$DBDIR"
 rm -f "$lock" || exit 1
 # echo 'done'
 
-- 
cgit v1.2.3-2-g168b


From ffa88b335b2f756925b388ecbd6681bf3aa58579 Mon Sep 17 00:00:00 2001
From: Dan McGee <dan@archlinux.org>
Date: Sat, 27 Feb 2010 11:52:56 -0600
Subject: create-filelists: include desc/depends entries

Make the files DB include everything the original packages DB includes
instead of just being 'files' entries. This will allow tools to do more with
these generated files and they can be used as a drop-in replacement for a
regular package database.

Signed-off-by: Dan McGee <dan@archlinux.org>
---
 cron-jobs/create-filelists | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

(limited to 'cron-jobs/create-filelists')

diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
index 6091bf4..84867d8 100755
--- a/cron-jobs/create-filelists
+++ b/cron-jobs/create-filelists
@@ -63,14 +63,18 @@ for repo in $repos; do
 
         # create file lists
         for pkg in $(ls ${DBDIR}/${repodir}); do
+            dbpkgdir="${DBDIR}/${repodir}/${pkg}"
+            cachepkgdir="${CACHEDIR}/${repodir}/${pkg}"
             tmppkgdir="${TMPDIR}/${repodir}/${pkg}"
             mkdir -p "$tmppkgdir"
-            if [ -f "${CACHEDIR}/${repodir}/${pkg}/files" ]; then
+            ln "${dbpkgdir}/desc" "${tmppkgdir}/desc"
+            ln "${dbpkgdir}/depends" "${tmppkgdir}/depends"
+            if [ -f "${cachepkgdir}/files" ]; then
 #               echo "cache: $pkg"
-                mv "${CACHEDIR}/${repodir}/${pkg}/files" "${tmppkgdir}/files"
+                ln "${cachepkgdir}/files" "${tmppkgdir}/files"
             else
 #               echo "not cache: $repo/$arch: $pkg"
-                filename=$(grep -A1 '^%FILENAME%$' "${DBDIR}/${repodir}/${pkg}/desc" | tail -n1)
+                filename=$(grep -A1 '^%FILENAME%$' "${dbpkgdir}/desc" | tail -n1)
                 echo '%FILES%' > "${tmppkgdir}/files"
                 bsdtar --exclude=.* -tf "$repodir/$filename" >> "${tmppkgdir}/files"
                 cached="no"
-- 
cgit v1.2.3-2-g168b


From 57987ae42fb58c58e044bfd7d3486010b32bb313 Mon Sep 17 00:00:00 2001
From: Pierre Schmitz <pierre@archlinux.de>
Date: Wed, 31 Mar 2010 23:09:57 +0200
Subject: don't publish *.old files

---
 cron-jobs/create-filelists | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

(limited to 'cron-jobs/create-filelists')

diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
index 84867d8..a25fd1a 100755
--- a/cron-jobs/create-filelists
+++ b/cron-jobs/create-filelists
@@ -88,8 +88,7 @@ for repo in $repos; do
             pkgdir="${targetdir}/${repodir}"
             mkdir -p "$pkgdir"
             cd "${TMPDIR}/${repodir}"
-            [ -f "${pkgdir}/${FILES_DB_FILE}.old" ] && rm "${pkgdir}/${FILES_DB_FILE}.old"
-            [ -f "${pkgdir}/${FILES_DB_FILE}" ] && mv "${pkgdir}/${FILES_DB_FILE}" "${pkgdir}/${FILES_DB_FILE}.old"
+            [ -f "${pkgdir}/${FILES_DB_FILE}" ] && rm "${pkgdir}/${FILES_DB_FILE}"
             bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f "${pkgdir}/${FILES_DB_FILE}" *
         fi
     done
-- 
cgit v1.2.3-2-g168b


From fdca04b3f5d8e27af4829416fee3843b0fe01e36 Mon Sep 17 00:00:00 2001
From: Pierre Schmitz <pierre@archlinux.de>
Date: Sun, 8 Aug 2010 16:43:16 +0200
Subject: Use db-functions in create-filelists and sourceballs

---
 cron-jobs/create-filelists | 50 ++++++++++++++++++----------------------------
 1 file changed, 19 insertions(+), 31 deletions(-)

(limited to 'cron-jobs/create-filelists')

diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
index a25fd1a..8c354fc 100755
--- a/cron-jobs/create-filelists
+++ b/cron-jobs/create-filelists
@@ -1,44 +1,38 @@
 #!/bin/bash
 
-reposdir="/srv/ftp"
-targetdir="/srv/ftp"
-repos="core extra testing community community-testing"
-lock="/tmp/create-filelists.lock"
-
 . "$(dirname $0)/../db-functions"
 . "$(dirname $0)/../config"
 
-if [ -f "$lock" ]; then
-    echo "Error: create-filelists already in progress."
-    exit 1
-fi
+reposdir=${FTP_BASE}
+targetdir=${FTP_BASE}
+repos=(core extra testing community community-testing)
+
+script_lock
 
-touch "$lock" || exit 1
 # location where the package DB is extracted so we know what to include
-DBDIR="$(mktemp -d /tmp/create-filelists.dbdir.XXXXXX)" || exit 1
+DBDIR="$(mktemp -d ${WORKDIR}/create-filelists.dbdir.XXXXXX)" || exit 1
 # location where the old files DB is extracted to save us some work
-CACHEDIR="$(mktemp -d /tmp/create-filelists.cachedir.XXXXXX)" || exit 1
+CACHEDIR="$(mktemp -d ${WORKDIR}/create-filelists.cachedir.XXXXXX)" || exit 1
 # location where the new files DB is built up and eventually zipped
-TMPDIR="$(mktemp -d /tmp/create-filelists.tmpdir.XXXXXX)" || exit 1
+MYTMPDIR="$(mktemp -d ${WORKDIR}/create-filelists.tmpdir.XXXXXX)" || exit 1
 
 #adjust the nice level to run at a lower priority
 /usr/bin/renice +10 -p $$ > /dev/null
 
 case "${DBEXT}" in
-    *.gz)  TAR_OPT="z" ;;
-    *.bz2) TAR_OPT="j" ;;
-    *.xz)  TAR_OPT="J" ;;
-    *) echo "Unknown compression type for DBEXT=${DBEXT}" && exit 1 ;;
+	*.gz)  TAR_OPT="z" ;;
+	*.bz2) TAR_OPT="j" ;;
+	*.xz)  TAR_OPT="J" ;;
+	*) echo "Unknown compression type for DBEXT=${DBEXT}" && exit 1 ;;
 esac
 
 FILESEXT="${DBEXT//db/files}"
 
-for repo in $repos; do
+for repo in ${repos[@]}; do
     REPO_DB_FILE="${repo}$DBEXT"
     FILES_DB_FILE="${repo}$FILESEXT"
     for arch in ${ARCHES[@]}; do
-#       echo "Running for architecture $arch, repo $repo"
-        cd "$reposdir"
+        pushd "$reposdir" >/dev/null
 
         repodir="${repo}/os/${arch}"
         cached="no"
@@ -46,7 +40,6 @@ for repo in $repos; do
         # extract package db archive
         if [ -f "${targetdir}/${repodir}/${REPO_DB_FILE}" ]; then
             mkdir -p "${DBDIR}/${repodir}"
-#           echo "extracting $REPO_DB_FILE"
             bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${DBDIR}/${repodir}"
         else
             echo "Fail! Does the repo $repo with arch $arch even exist?"
@@ -56,7 +49,6 @@ for repo in $repos; do
         # extract old file archive
         if [ -f "${targetdir}/${repodir}/${FILES_DB_FILE}" ]; then
             mkdir -p "${CACHEDIR}/${repodir}"
-#           echo "extracting $FILES_DB_FILE"
             bsdtar -xf "${targetdir}/${repodir}/${FILES_DB_FILE}" -C "${CACHEDIR}/${repodir}"
             cached="yes"
         fi
@@ -65,15 +57,13 @@ for repo in $repos; do
         for pkg in $(ls ${DBDIR}/${repodir}); do
             dbpkgdir="${DBDIR}/${repodir}/${pkg}"
             cachepkgdir="${CACHEDIR}/${repodir}/${pkg}"
-            tmppkgdir="${TMPDIR}/${repodir}/${pkg}"
+            tmppkgdir="${MYTMPDIR}/${repodir}/${pkg}"
             mkdir -p "$tmppkgdir"
             ln "${dbpkgdir}/desc" "${tmppkgdir}/desc"
             ln "${dbpkgdir}/depends" "${tmppkgdir}/depends"
             if [ -f "${cachepkgdir}/files" ]; then
-#               echo "cache: $pkg"
                 ln "${cachepkgdir}/files" "${tmppkgdir}/files"
             else
-#               echo "not cache: $repo/$arch: $pkg"
                 filename=$(grep -A1 '^%FILENAME%$' "${dbpkgdir}/desc" | tail -n1)
                 echo '%FILES%' > "${tmppkgdir}/files"
                 bsdtar --exclude=.* -tf "$repodir/$filename" >> "${tmppkgdir}/files"
@@ -84,19 +74,17 @@ for repo in $repos; do
         # create new file archive
         if [ "$cached" == "no" ]; then
             # at least one package has changed, so let's rebuild the archive
-#           echo "creating ${FILES_DB_FILE}/${arch}"
             pkgdir="${targetdir}/${repodir}"
             mkdir -p "$pkgdir"
-            cd "${TMPDIR}/${repodir}"
+            pushd "${MYTMPDIR}/${repodir}" >/dev/null
             [ -f "${pkgdir}/${FILES_DB_FILE}" ] && rm "${pkgdir}/${FILES_DB_FILE}"
             bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f "${pkgdir}/${FILES_DB_FILE}" *
+            popd >/dev/null
         fi
+        popd >/dev/null
     done
 done
 
-cd - >/dev/null
-rm -rf "$TMPDIR" "$CACHEDIR" "$DBDIR"
-rm -f "$lock" || exit 1
-# echo 'done'
+script_unlock
 
 # vim: set ts=4 sw=4 et ft=sh:
-- 
cgit v1.2.3-2-g168b


From a422060414670bb49d2422a38467b73ae01e7ecb Mon Sep 17 00:00:00 2001
From: Pierre Schmitz <pierre@archlinux.de>
Date: Fri, 13 Aug 2010 09:47:31 +0200
Subject: Use common functions to print messages, warnings and errors

These functions are copied from makepkg
---
 cron-jobs/create-filelists | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'cron-jobs/create-filelists')

diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
index 8c354fc..9249408 100755
--- a/cron-jobs/create-filelists
+++ b/cron-jobs/create-filelists
@@ -23,7 +23,7 @@ case "${DBEXT}" in
 	*.gz)  TAR_OPT="z" ;;
 	*.bz2) TAR_OPT="j" ;;
 	*.xz)  TAR_OPT="J" ;;
-	*) echo "Unknown compression type for DBEXT=${DBEXT}" && exit 1 ;;
+	*) die "Unknown compression type for DBEXT=${DBEXT}" ;;
 esac
 
 FILESEXT="${DBEXT//db/files}"
@@ -42,7 +42,7 @@ for repo in ${repos[@]}; do
             mkdir -p "${DBDIR}/${repodir}"
             bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${DBDIR}/${repodir}"
         else
-            echo "Fail! Does the repo $repo with arch $arch even exist?"
+            error "Fail! Does the repo $repo with arch $arch even exist?"
             continue
         fi
 
-- 
cgit v1.2.3-2-g168b


From 4daeb0a71e65618c56d672118675c9efd57c5979 Mon Sep 17 00:00:00 2001
From: Pierre Schmitz <pierre@archlinux.de>
Date: Tue, 17 Aug 2010 08:00:21 +0200
Subject: Add [staging] repository

---
 cron-jobs/create-filelists | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'cron-jobs/create-filelists')

diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
index 9249408..fd07617 100755
--- a/cron-jobs/create-filelists
+++ b/cron-jobs/create-filelists
@@ -5,7 +5,7 @@
 
 reposdir=${FTP_BASE}
 targetdir=${FTP_BASE}
-repos=(core extra testing community community-testing)
+repos=(core extra testing community community-testing staging)
 
 script_lock
 
-- 
cgit v1.2.3-2-g168b


From 9e8897aeadb2aa51b70c5fc0ed1ae281b015d146 Mon Sep 17 00:00:00 2001
From: Pierre Schmitz <pierre@archlinux.de>
Date: Sun, 22 Aug 2010 22:01:09 +0200
Subject: add makepkg.conf for [multilib]

---
 cron-jobs/create-filelists | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'cron-jobs/create-filelists')

diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
index fd07617..e280f84 100755
--- a/cron-jobs/create-filelists
+++ b/cron-jobs/create-filelists
@@ -5,7 +5,7 @@
 
 reposdir=${FTP_BASE}
 targetdir=${FTP_BASE}
-repos=(core extra testing community community-testing staging)
+repos=(core extra testing community community-testing staging multilib)
 
 script_lock
 
-- 
cgit v1.2.3-2-g168b


From 328b1ce478e25902aba5d8f19024dadeaaf3f678 Mon Sep 17 00:00:00 2001
From: Pierre Schmitz <pierre@archlinux.de>
Date: Wed, 1 Sep 2010 19:34:47 +0200
Subject: Simplify repo configuration

* Repositories can now be defined in the config file for each host
* added community-staging, gnome-unstable and kde-unstable
* Exception is the adjust-permission cron-job; but we might want to use acls in future anyway
Signed-off-by: Pierre Schmitz <pierre@archlinux.de>
---
 cron-jobs/create-filelists | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

(limited to 'cron-jobs/create-filelists')

diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
index e280f84..a37080f 100755
--- a/cron-jobs/create-filelists
+++ b/cron-jobs/create-filelists
@@ -5,7 +5,6 @@
 
 reposdir=${FTP_BASE}
 targetdir=${FTP_BASE}
-repos=(core extra testing community community-testing staging multilib)
 
 script_lock
 
@@ -28,7 +27,7 @@ esac
 
 FILESEXT="${DBEXT//db/files}"
 
-for repo in ${repos[@]}; do
+for repo in ${PKGREPO[@]}; do
     REPO_DB_FILE="${repo}$DBEXT"
     FILES_DB_FILE="${repo}$FILESEXT"
     for arch in ${ARCHES[@]}; do
-- 
cgit v1.2.3-2-g168b


From d9efa9b7bbbc854b987f158cd19d5f646594e42b Mon Sep 17 00:00:00 2001
From: Pierre Schmitz <pierre@archlinux.de>
Date: Mon, 13 Sep 2010 23:08:20 +0200
Subject: Fix typos

---
 cron-jobs/create-filelists | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'cron-jobs/create-filelists')

diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
index a37080f..2526a61 100755
--- a/cron-jobs/create-filelists
+++ b/cron-jobs/create-filelists
@@ -27,7 +27,7 @@ esac
 
 FILESEXT="${DBEXT//db/files}"
 
-for repo in ${PKGREPO[@]}; do
+for repo in ${PKGREPOS[@]}; do
     REPO_DB_FILE="${repo}$DBEXT"
     FILES_DB_FILE="${repo}$FILESEXT"
     for arch in ${ARCHES[@]}; do
-- 
cgit v1.2.3-2-g168b


From a1ba979a0e3c763449d91fc787333f4058e89aa8 Mon Sep 17 00:00:00 2001
From: Pierre Schmitz <pierre@archlinux.de>
Date: Tue, 12 Oct 2010 11:17:07 +0200
Subject: Dont try to index empty repos

---
 cron-jobs/create-filelists | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'cron-jobs/create-filelists')

diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
index 2526a61..d8a5ba1 100755
--- a/cron-jobs/create-filelists
+++ b/cron-jobs/create-filelists
@@ -41,7 +41,7 @@ for repo in ${PKGREPOS[@]}; do
             mkdir -p "${DBDIR}/${repodir}"
             bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${DBDIR}/${repodir}"
         else
-            error "Fail! Does the repo $repo with arch $arch even exist?"
+            warning "The repo $repo with arch $arch does not exist"
             continue
         fi
 
@@ -71,7 +71,7 @@ for repo in ${PKGREPOS[@]}; do
         done
 
         # create new file archive
-        if [ "$cached" == "no" ]; then
+        if [ "$cached" == "no" -a -d "${MYTMPDIR}/${repodir}" ]; then
             # at least one package has changed, so let's rebuild the archive
             pkgdir="${targetdir}/${repodir}"
             mkdir -p "$pkgdir"
-- 
cgit v1.2.3-2-g168b


From 447986d17554c183d958cf7de79d1904fc32aaf6 Mon Sep 17 00:00:00 2001
From: Pierre Schmitz <pierre@archlinux.de>
Date: Thu, 25 Nov 2010 13:28:04 +0100
Subject: Add FILESEXT variable for use in create-filelists

---
 cron-jobs/create-filelists | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

(limited to 'cron-jobs/create-filelists')

diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
index d8a5ba1..010194b 100755
--- a/cron-jobs/create-filelists
+++ b/cron-jobs/create-filelists
@@ -18,15 +18,13 @@ MYTMPDIR="$(mktemp -d ${WORKDIR}/create-filelists.tmpdir.XXXXXX)" || exit 1
 #adjust the nice level to run at a lower priority
 /usr/bin/renice +10 -p $$ > /dev/null
 
-case "${DBEXT}" in
+case "${FILESEXT}" in
 	*.gz)  TAR_OPT="z" ;;
 	*.bz2) TAR_OPT="j" ;;
 	*.xz)  TAR_OPT="J" ;;
-	*) die "Unknown compression type for DBEXT=${DBEXT}" ;;
+	*) die "Unknown compression type for FILESEXT=${FILESEXT}" ;;
 esac
 
-FILESEXT="${DBEXT//db/files}"
-
 for repo in ${PKGREPOS[@]}; do
     REPO_DB_FILE="${repo}$DBEXT"
     FILES_DB_FILE="${repo}$FILESEXT"
@@ -77,7 +75,7 @@ for repo in ${PKGREPOS[@]}; do
             mkdir -p "$pkgdir"
             pushd "${MYTMPDIR}/${repodir}" >/dev/null
             [ -f "${pkgdir}/${FILES_DB_FILE}" ] && rm "${pkgdir}/${FILES_DB_FILE}"
-            bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f "${pkgdir}/${FILES_DB_FILE}" *
+            bsdtar --exclude=*${FILESEXT} -c${TAR_OPT}f "${pkgdir}/${FILES_DB_FILE}" *
             popd >/dev/null
         fi
         popd >/dev/null
-- 
cgit v1.2.3-2-g168b


From e792abefa6eff4cf54616add0b19d731d486045f Mon Sep 17 00:00:00 2001
From: Pierre Schmitz <pierre@archlinux.de>
Date: Thu, 25 Nov 2010 16:03:44 +0100
Subject: Rewrote create-filelists

* use correct locking of the repos
* added test case
* removes file lists of deleted packages
* add compression independent symlink to files db
---
 cron-jobs/create-filelists | 147 +++++++++++++++++++++++++--------------------
 1 file changed, 81 insertions(+), 66 deletions(-)

(limited to 'cron-jobs/create-filelists')

diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
index 010194b..c3e7d72 100755
--- a/cron-jobs/create-filelists
+++ b/cron-jobs/create-filelists
@@ -3,20 +3,42 @@
 . "$(dirname $0)/../db-functions"
 . "$(dirname $0)/../config"
 
-reposdir=${FTP_BASE}
-targetdir=${FTP_BASE}
-
 script_lock
 
-# location where the package DB is extracted so we know what to include
-DBDIR="$(mktemp -d ${WORKDIR}/create-filelists.dbdir.XXXXXX)" || exit 1
-# location where the old files DB is extracted to save us some work
-CACHEDIR="$(mktemp -d ${WORKDIR}/create-filelists.cachedir.XXXXXX)" || exit 1
-# location where the new files DB is built up and eventually zipped
-MYTMPDIR="$(mktemp -d ${WORKDIR}/create-filelists.tmpdir.XXXXXX)" || exit 1
+for repo in ${PKGREPOS[@]}; do
+	for arch in ${ARCHES[@]}; do
+		repo_lock ${repo} ${arch} || exit 1
+	done
+done
 
 #adjust the nice level to run at a lower priority
-/usr/bin/renice +10 -p $$ > /dev/null
+renice +10 -p $$ > /dev/null
+
+for repo in ${PKGREPOS[@]}; do
+	for arch in ${ARCHES[@]}; do
+		repodb="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}"
+		filedb="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}"
+
+		if [ ! -f "${repodb}" ]; then
+			continue
+		fi
+		# get a list of package files defined in the repo db
+		mkdir -p "${WORKDIR}/db-dir-${repo}-${arch}"
+		bsdtar -xf "${repodb}" -C "${WORKDIR}/db-dir-${repo}-${arch}"
+		find "${WORKDIR}/db-dir-${repo}-${arch}" -name 'desc' \
+			-exec awk '/^%FILENAME%/{getline;print}' {} \; | sort > "${WORKDIR}/db-${repo}-${arch}"
+
+		# get a list of package files defined in the files db
+		mkdir -p "${WORKDIR}/files-current-dir-${repo}-${arch}"
+		if [ ! -f "${filedb}" ]; then
+			echo > "${WORKDIR}/files-${repo}-${arch}"
+		else
+			bsdtar -xf "${filedb}" -C "${WORKDIR}/files-current-dir-${repo}-${arch}"
+			find "${WORKDIR}/files-current-dir-${repo}-${arch}" -name 'desc' \
+				-exec awk '/^%FILENAME%/{getline;print}' {} \; | sort > "${WORKDIR}/files-${repo}-${arch}"
+		fi
+	done
+done
 
 case "${FILESEXT}" in
 	*.gz)  TAR_OPT="z" ;;
@@ -26,62 +48,55 @@ case "${FILESEXT}" in
 esac
 
 for repo in ${PKGREPOS[@]}; do
-    REPO_DB_FILE="${repo}$DBEXT"
-    FILES_DB_FILE="${repo}$FILESEXT"
-    for arch in ${ARCHES[@]}; do
-        pushd "$reposdir" >/dev/null
-
-        repodir="${repo}/os/${arch}"
-        cached="no"
-
-        # extract package db archive
-        if [ -f "${targetdir}/${repodir}/${REPO_DB_FILE}" ]; then
-            mkdir -p "${DBDIR}/${repodir}"
-            bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${DBDIR}/${repodir}"
-        else
-            warning "The repo $repo with arch $arch does not exist"
-            continue
-        fi
-
-        # extract old file archive
-        if [ -f "${targetdir}/${repodir}/${FILES_DB_FILE}" ]; then
-            mkdir -p "${CACHEDIR}/${repodir}"
-            bsdtar -xf "${targetdir}/${repodir}/${FILES_DB_FILE}" -C "${CACHEDIR}/${repodir}"
-            cached="yes"
-        fi
-
-        # create file lists
-        for pkg in $(ls ${DBDIR}/${repodir}); do
-            dbpkgdir="${DBDIR}/${repodir}/${pkg}"
-            cachepkgdir="${CACHEDIR}/${repodir}/${pkg}"
-            tmppkgdir="${MYTMPDIR}/${repodir}/${pkg}"
-            mkdir -p "$tmppkgdir"
-            ln "${dbpkgdir}/desc" "${tmppkgdir}/desc"
-            ln "${dbpkgdir}/depends" "${tmppkgdir}/depends"
-            if [ -f "${cachepkgdir}/files" ]; then
-                ln "${cachepkgdir}/files" "${tmppkgdir}/files"
-            else
-                filename=$(grep -A1 '^%FILENAME%$' "${dbpkgdir}/desc" | tail -n1)
-                echo '%FILES%' > "${tmppkgdir}/files"
-                bsdtar --exclude=.* -tf "$repodir/$filename" >> "${tmppkgdir}/files"
-                cached="no"
-            fi
-        done
-
-        # create new file archive
-        if [ "$cached" == "no" -a -d "${MYTMPDIR}/${repodir}" ]; then
-            # at least one package has changed, so let's rebuild the archive
-            pkgdir="${targetdir}/${repodir}"
-            mkdir -p "$pkgdir"
-            pushd "${MYTMPDIR}/${repodir}" >/dev/null
-            [ -f "${pkgdir}/${FILES_DB_FILE}" ] && rm "${pkgdir}/${FILES_DB_FILE}"
-            bsdtar --exclude=*${FILESEXT} -c${TAR_OPT}f "${pkgdir}/${FILES_DB_FILE}" *
-            popd >/dev/null
-        fi
-        popd >/dev/null
-    done
+	for arch in ${ARCHES[@]}; do
+		filedb="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}"
+
+		if [ ! -f  "${WORKDIR}/db-${repo}-${arch}" ]; then
+			# remove any files db that might be in this empty repo
+			if [ -f "${filedb}" ]; then
+				rm -f "${filedb}"
+			fi
+			continue
+		fi
+
+		# Check if updating the files db is needed
+		if ! diff -q "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}" >/dev/null; then
+			mkdir -p "${WORKDIR}/files-new-dir-${repo}-${arch}"
+
+			# Include all unchanged file lists
+			# Note: deleted packages are implicitly excluded
+			for f in $(comm -12 "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}"); do
+				mv "${WORKDIR}/files-current-dir-${repo}-${arch}/${f%*-*${PKGEXT}}" \
+					"${WORKDIR}/files-new-dir-${repo}-${arch}"
+			done
+
+			# Create file lists for new packages
+			for f in $(comm -23 "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}"); do
+				tdir="${WORKDIR}/files-new-dir-${repo}-${arch}/${f%*-*${PKGEXT}}"
+				mkdir "${tdir}"
+				echo '%FILES%' > "${tdir}/files"
+				bsdtar --exclude=.* -tf "${FTP_BASE}/${repo}/os/${arch}/${f}" >> "${tdir}/files"
+
+				# add desc and depends file from db
+				dbdir="${WORKDIR}/db-dir-${repo}-${arch}/${f%*-*${PKGEXT}}"
+				mv "${dbdir}/desc" "${tdir}/desc"
+				mv "${dbdir}/depends" "${tdir}/depends"
+			done
+
+			# Create the actual file db
+			pushd "${WORKDIR}/files-new-dir-${repo}-${arch}" >/dev/null
+			bsdtar -c${TAR_OPT}f "${WORKDIR}/${arch}-${repo}${FILESEXT}" *
+			popd >/dev/null
+			mv -f "${WORKDIR}/${arch}-${repo}${FILESEXT}" "${filedb}"
+			ln -sf "${repo}${FILESEXT}" "${filedb%.tar.*}"
+		fi
+	done
+done
+
+for repo in ${PKGREPOS[@]}; do
+	for arch in ${ARCHES[@]}; do
+		repo_unlock ${repo} ${arch}
+	done
 done
 
 script_unlock
-
-# vim: set ts=4 sw=4 et ft=sh:
-- 
cgit v1.2.3-2-g168b


From 88a00e08e0394c95f65836be043da75fa2a131f1 Mon Sep 17 00:00:00 2001
From: Pierre Schmitz <pierre@archlinux.de>
Date: Thu, 25 Nov 2010 19:05:53 +0100
Subject: create-filelists: Improve performance by reading package list from db
 instead from fs

---
 cron-jobs/create-filelists | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

(limited to 'cron-jobs/create-filelists')

diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists
index c3e7d72..8bcfc0b 100755
--- a/cron-jobs/create-filelists
+++ b/cron-jobs/create-filelists
@@ -25,17 +25,16 @@ for repo in ${PKGREPOS[@]}; do
 		# get a list of package files defined in the repo db
 		mkdir -p "${WORKDIR}/db-dir-${repo}-${arch}"
 		bsdtar -xf "${repodb}" -C "${WORKDIR}/db-dir-${repo}-${arch}"
-		find "${WORKDIR}/db-dir-${repo}-${arch}" -name 'desc' \
-			-exec awk '/^%FILENAME%/{getline;print}' {} \; | sort > "${WORKDIR}/db-${repo}-${arch}"
+		# This should actualy be faster than reading all the just extracted files
+		bsdtar -xOf "${repodb}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/db-${repo}-${arch}"
 
 		# get a list of package files defined in the files db
 		mkdir -p "${WORKDIR}/files-current-dir-${repo}-${arch}"
 		if [ ! -f "${filedb}" ]; then
-			echo > "${WORKDIR}/files-${repo}-${arch}"
+			touch "${WORKDIR}/files-${repo}-${arch}"
 		else
 			bsdtar -xf "${filedb}" -C "${WORKDIR}/files-current-dir-${repo}-${arch}"
-			find "${WORKDIR}/files-current-dir-${repo}-${arch}" -name 'desc' \
-				-exec awk '/^%FILENAME%/{getline;print}' {} \; | sort > "${WORKDIR}/files-${repo}-${arch}"
+			bsdtar -xOf "${filedb}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/files-${repo}-${arch}"
 		fi
 	done
 done
-- 
cgit v1.2.3-2-g168b