aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAndreas K. Hüttel <dilfridge@gentoo.org>2023-08-06 23:40:02 +0200
committerAndreas K. Hüttel <dilfridge@gentoo.org>2023-08-06 23:40:02 +0200
commit2fe2355ae0ccf4067d685adbadefba964289737e (patch)
tree1067b8f87db58bcf772a91f9fdb508aea5bb6686
parentSome comments (diff)
downloadmastermirror-scripts-2fe2355ae0ccf4067d685adbadefba964289737e.tar.gz
mastermirror-scripts-2fe2355ae0ccf4067d685adbadefba964289737e.tar.bz2
mastermirror-scripts-2fe2355ae0ccf4067d685adbadefba964289737e.zip
Merge two scripts and speed things up
Signed-off-by: Andreas K. Hüttel <dilfridge@gentoo.org>
-rwxr-xr-xsign-binpackages.sh51
-rwxr-xr-xsign-sync-binpackages.sh102
-rwxr-xr-xsync-binpackages.sh38
3 files changed, 102 insertions, 89 deletions
diff --git a/sign-binpackages.sh b/sign-binpackages.sh
deleted file mode 100755
index 61d8874..0000000
--- a/sign-binpackages.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/bin/bash
-# Copyright 2023 Gentoo Authors; Distributed under the GPL v2
-# might be earlier copyright, no history available
-
-# NOTE 1: This script is SLOW. It should run at most once per day.
-# NOTE 2: This script requires that the signing key has its ownertrust
-# set to ultimate. Which makes sense anyway, since we have the
-# secret key.
-# NOTE 3: This script has to run as gmirror user.
-
-# for testing
-ARCHES="sparc"
-
-# Keep this variable in sync
-_ARCHES="alpha amd64 arm64 arm hppa ia64 loong m68k mips ppc riscv s390 sh sparc x86"
- #alpha amd64 arm64 arm hppa ia64 loong m68k mips ppc riscv s390 sh sparc x86
-ARCHES=${ARCHES:-${_ARCHES}}
-
-
-INTREE=/release/weekly/binpackages
-SIGTREE=/release/binpackages
-
-export BINPKG_GPG_SIGNING_GPG_HOME=/home/gmirror/.gnupg-releng
-export BINPKG_GPG_SIGNING_KEY=13EBBDBEDE7A12775DFDB1BABB572E0E2D182910
-export BINPKG_GPG_VERIFY_GPG_HOME=${BINPKG_GPG_SIGNING_GPG_HOME}
-
-[[ $(whoami) == "gmirror" ]] || exit 111
-
-gpgconf --kill all
-
-# step 1: rsync from the dirs where the arches copy in
-# make sure to *not* overwrite existing newer files (obviously
-# the signature changed them)...
-
-for a in $ARCHES ; do
- rsync --archive --delete --delete-after --update --mkpath ${INTREE}/${a}/* ${SIGTREE}/${a}/
-done
-
-# step 2: iterate over all binary package trees, sign
-# all unsigned files
-# we assume the directory structure to be
-# .../binpackages/amd64/17.1/x86-64
-# .../binpackages/amd64/17.1/x86-64_musl
-# .../binpackages/mips/17.0/mipsel3_n32
-# .../binpackages/x86/17.0/x86_musl_hardened
-
-for t in ${SIGTREE}/*/*/* ; do
- find "${t}" -name '*.gpkg.tar' -exec gpkg-sign --skip-signed \{\} \; > /dev/null
- PKGDIR=${t} emaint -f binhost > /dev/null
-done
-# unfortunately these commands make much noise... let's hope we notice errors
diff --git a/sign-sync-binpackages.sh b/sign-sync-binpackages.sh
new file mode 100755
index 0000000..d5e1caf
--- /dev/null
+++ b/sign-sync-binpackages.sh
@@ -0,0 +1,102 @@
+#!/bin/bash
+# Copyright 2023 Gentoo Authors; Distributed under the GPL v2
+# might be earlier copyright, no history available
+
+# NOTE 1: This script is SLOW. It should run at most once per day.
+# NOTE 2: This script requires that the signing key has its ownertrust
+# set to ultimate. Which makes sense anyway, since we have the
+# secret key.
+# NOTE 3: This script has to run as gmirror user.
+
+# for testing
+ARCHES="sparc"
+
+# Keep this variable in sync
+_ARCHES="alpha amd64 arm64 arm hppa ia64 loong m68k mips ppc riscv s390 sh sparc x86"
+ #alpha amd64 arm64 arm hppa ia64 loong m68k mips ppc riscv s390 sh sparc x86
+ARCHES=${ARCHES:-${_ARCHES}}
+
+
+INTREE=/release/weekly/binpackages
+STAGINGTREE=/release/binpackages-staging
+OUTTREE=/var/tmp/gmirror-releases/releases
+
+IN_RSYNC_OPTS=(
+ --archive
+ --delete
+ --delete-after
+ --update
+ --mkpath
+)
+
+OUT_RSYNC_OPTS=(
+ --no-motd
+ --archive
+ --ignore-errors
+ --delete
+ --delete-after
+ --timeout=300
+ --mkpath
+)
+
+export BINPKG_GPG_SIGNING_GPG_HOME=/home/gmirror/.gnupg-releng
+export BINPKG_GPG_SIGNING_KEY=13EBBDBEDE7A12775DFDB1BABB572E0E2D182910
+export BINPKG_GPG_VERIFY_GPG_HOME=${BINPKG_GPG_SIGNING_GPG_HOME}
+
+# this script needs to be run as gmirror user
+[[ $(whoami) == "gmirror" ]] || exit 111
+
+# we make sure we're not running twice in parallel
+[[ -f ${STAGINGTREE}/.running ]] && exit 112
+touch ${STAGINGTREE}/.running || exit 110
+
+# make sure we have an updated gpg-agent
+gpgconf --kill all
+
+
+# step 1: rsync from the dirs where the arches copy in
+# make sure to *not* overwrite existing newer files (obviously
+# the signature changed them)...
+
+for a in ${ARCHES} ; do
+ rsync "${IN_RSYNC_OPTS[@]}" ${INTREE}/${a}/* ${STAGINGTREE}/${a}/
+done
+
+# now the set of files is frozen in the staging dir, and we dont care
+# if any arches start uploading in the meantime
+
+
+# step 2: iterate over all binary package trees, sign
+# all unsigned files
+# we assume the directory structure to be
+# .../binpackages-staging/amd64/17.1/x86-64
+# .../binpackages-staging/amd64/17.1/x86-64_musl
+# .../binpackages-staging/mips/17.0/mipsel3_n32
+# .../binpackages-staging/x86/17.0/x86_musl_hardened
+
+for t in ${STAGINGTREE}/*/*/* ; do
+ # find all unsigned packages as fast as possible
+ find "${t}" -name '*.gpkg.tar' -print0 | \
+ parallel -0 -n1 -- "tar tf {} |grep -E -e '/metadata\.tar\..*\.sig$' -L --label={}" > ${STAGINGTREE}/.unsigned
+
+ # sign the packages
+ xargs -n1 -- gpkg-sign "{}" < ${STAGINGTREE}/.unsigned > /dev/null || exit 113
+
+ # regenerate the indices
+ PKGDIR=${t} emaint -f binhost > /dev/null || exit 114
+done
+# unfortunately these commands make much noise... let's hope we notice errors
+
+
+# step 3: sync the result into the mirror directories from where
+# the files are distributed
+
+for a in ${ARCHES}; do
+ [[ -d ${OUTTREE}/${a}/binpackages ]] || mkdir -p ${OUTTREE}/${a}/binpackages
+ rsync "${OUT_RSYNC_OPTS[@]}" ${STAGINGTREE}/${a}/* ${OUTTREE}/${a}/binpackages/
+ date -u > ${OUTTREE}/${a}/binpackages/.timestamp
+done
+
+
+# we're done so remove the "lockfile"
+rm ${STAGINGTREE}/.running
diff --git a/sync-binpackages.sh b/sync-binpackages.sh
deleted file mode 100755
index fe524d9..0000000
--- a/sync-binpackages.sh
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/bin/bash
-# Copyright 2011-2023 Gentoo Authors; Distributed under the GPL v2
-
-# for testing
-ARCHES="sparc"
-
-# Keep this variable in sync in both sign-binpackages.sh & sync-binpackages.sh
-_ARCHES="alpha amd64 arm64 arm hppa ia64 loong m68k mips ppc riscv s390 sh sparc x86"
- #alpha amd64 arm64 arm hppa ia64 loong m68k mips ppc riscv s390 sh sparc x86
-ARCHES=${ARCHES:-${_ARCHES}}
-
-# this is the directory where all packages are signed
-# we assume it's on dipper / releng-incoming, but might as well give a full rsync
-# specification here
-SRCDIR="/release/binpackages"
-# append ${a}
-
-# this is the outgoing directory
-DSTDIR="/var/tmp/gmirror-releases/releases"
-# append ${a}/binpackages
-
-RSYNC="/usr/bin/rsync"
-RSYNC_OPTS=(
- --no-motd
- --archive
- --ignore-errors
- --delete
- --delete-after
- --timeout=300
- --mkpath
-)
-
-[[ $(whoami) == "gmirror" ]] || exit 111
-
-for a in ${ARCHES}; do
- [[ -d ${DSTDIR}/${a}/binpackages ]] || mkdir -p ${DSTDIR}/${a}/binpackages
- rsync "${RSYNC_OPTS[@]}" ${SRCDIR}/${a}/* ${DSTDIR}/${a}/binpackages/
-done