diff options
author | 2023-08-06 23:40:02 +0200 | |
---|---|---|
committer | 2023-08-06 23:40:02 +0200 | |
commit | 2fe2355ae0ccf4067d685adbadefba964289737e (patch) | |
tree | 1067b8f87db58bcf772a91f9fdb508aea5bb6686 | |
parent | Some comments (diff) | |
download | mastermirror-scripts-2fe2355ae0ccf4067d685adbadefba964289737e.tar.gz mastermirror-scripts-2fe2355ae0ccf4067d685adbadefba964289737e.tar.bz2 mastermirror-scripts-2fe2355ae0ccf4067d685adbadefba964289737e.zip |
Merge two scripts and speed things up
Signed-off-by: Andreas K. Hüttel <dilfridge@gentoo.org>
-rwxr-xr-x | sign-binpackages.sh | 51 | ||||
-rwxr-xr-x | sign-sync-binpackages.sh | 102 | ||||
-rwxr-xr-x | sync-binpackages.sh | 38 |
3 files changed, 102 insertions, 89 deletions
diff --git a/sign-binpackages.sh b/sign-binpackages.sh deleted file mode 100755 index 61d8874..0000000 --- a/sign-binpackages.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -# Copyright 2023 Gentoo Authors; Distributed under the GPL v2 -# might be earlier copyright, no history available - -# NOTE 1: This script is SLOW. It should run at most once per day. -# NOTE 2: This script requires that the signing key has its ownertrust -# set to ultimate. Which makes sense anyway, since we have the -# secret key. -# NOTE 3: This script has to run as gmirror user. - -# for testing -ARCHES="sparc" - -# Keep this variable in sync -_ARCHES="alpha amd64 arm64 arm hppa ia64 loong m68k mips ppc riscv s390 sh sparc x86" - #alpha amd64 arm64 arm hppa ia64 loong m68k mips ppc riscv s390 sh sparc x86 -ARCHES=${ARCHES:-${_ARCHES}} - - -INTREE=/release/weekly/binpackages -SIGTREE=/release/binpackages - -export BINPKG_GPG_SIGNING_GPG_HOME=/home/gmirror/.gnupg-releng -export BINPKG_GPG_SIGNING_KEY=13EBBDBEDE7A12775DFDB1BABB572E0E2D182910 -export BINPKG_GPG_VERIFY_GPG_HOME=${BINPKG_GPG_SIGNING_GPG_HOME} - -[[ $(whoami) == "gmirror" ]] || exit 111 - -gpgconf --kill all - -# step 1: rsync from the dirs where the arches copy in -# make sure to *not* overwrite existing newer files (obviously -# the signature changed them)... - -for a in $ARCHES ; do - rsync --archive --delete --delete-after --update --mkpath ${INTREE}/${a}/* ${SIGTREE}/${a}/ -done - -# step 2: iterate over all binary package trees, sign -# all unsigned files -# we assume the directory structure to be -# .../binpackages/amd64/17.1/x86-64 -# .../binpackages/amd64/17.1/x86-64_musl -# .../binpackages/mips/17.0/mipsel3_n32 -# .../binpackages/x86/17.0/x86_musl_hardened - -for t in ${SIGTREE}/*/*/* ; do - find "${t}" -name '*.gpkg.tar' -exec gpkg-sign --skip-signed \{\} \; > /dev/null - PKGDIR=${t} emaint -f binhost > /dev/null -done -# unfortunately these commands make much noise... let's hope we notice errors diff --git a/sign-sync-binpackages.sh b/sign-sync-binpackages.sh new file mode 100755 index 0000000..d5e1caf --- /dev/null +++ b/sign-sync-binpackages.sh @@ -0,0 +1,102 @@ +#!/bin/bash +# Copyright 2023 Gentoo Authors; Distributed under the GPL v2 +# might be earlier copyright, no history available + +# NOTE 1: This script is SLOW. It should run at most once per day. +# NOTE 2: This script requires that the signing key has its ownertrust +# set to ultimate. Which makes sense anyway, since we have the +# secret key. +# NOTE 3: This script has to run as gmirror user. + +# for testing +ARCHES="sparc" + +# Keep this variable in sync +_ARCHES="alpha amd64 arm64 arm hppa ia64 loong m68k mips ppc riscv s390 sh sparc x86" + #alpha amd64 arm64 arm hppa ia64 loong m68k mips ppc riscv s390 sh sparc x86 +ARCHES=${ARCHES:-${_ARCHES}} + + +INTREE=/release/weekly/binpackages +STAGINGTREE=/release/binpackages-staging +OUTTREE=/var/tmp/gmirror-releases/releases + +IN_RSYNC_OPTS=( + --archive + --delete + --delete-after + --update + --mkpath +) + +OUT_RSYNC_OPTS=( + --no-motd + --archive + --ignore-errors + --delete + --delete-after + --timeout=300 + --mkpath +) + +export BINPKG_GPG_SIGNING_GPG_HOME=/home/gmirror/.gnupg-releng +export BINPKG_GPG_SIGNING_KEY=13EBBDBEDE7A12775DFDB1BABB572E0E2D182910 +export BINPKG_GPG_VERIFY_GPG_HOME=${BINPKG_GPG_SIGNING_GPG_HOME} + +# this script needs to be run as gmirror user +[[ $(whoami) == "gmirror" ]] || exit 111 + +# we make sure we're not running twice in parallel +[[ -f ${STAGINGTREE}/.running ]] && exit 112 +touch ${STAGINGTREE}/.running || exit 110 + +# make sure we have an updated gpg-agent +gpgconf --kill all + + +# step 1: rsync from the dirs where the arches copy in +# make sure to *not* overwrite existing newer files (obviously +# the signature changed them)... + +for a in ${ARCHES} ; do + rsync "${IN_RSYNC_OPTS[@]}" ${INTREE}/${a}/* ${STAGINGTREE}/${a}/ +done + +# now the set of files is frozen in the staging dir, and we dont care +# if any arches start uploading in the meantime + + +# step 2: iterate over all binary package trees, sign +# all unsigned files +# we assume the directory structure to be +# .../binpackages-staging/amd64/17.1/x86-64 +# .../binpackages-staging/amd64/17.1/x86-64_musl +# .../binpackages-staging/mips/17.0/mipsel3_n32 +# .../binpackages-staging/x86/17.0/x86_musl_hardened + +for t in ${STAGINGTREE}/*/*/* ; do + # find all unsigned packages as fast as possible + find "${t}" -name '*.gpkg.tar' -print0 | \ + parallel -0 -n1 -- "tar tf {} |grep -E -e '/metadata\.tar\..*\.sig$' -L --label={}" > ${STAGINGTREE}/.unsigned + + # sign the packages + xargs -n1 -- gpkg-sign "{}" < ${STAGINGTREE}/.unsigned > /dev/null || exit 113 + + # regenerate the indices + PKGDIR=${t} emaint -f binhost > /dev/null || exit 114 +done +# unfortunately these commands make much noise... let's hope we notice errors + + +# step 3: sync the result into the mirror directories from where +# the files are distributed + +for a in ${ARCHES}; do + [[ -d ${OUTTREE}/${a}/binpackages ]] || mkdir -p ${OUTTREE}/${a}/binpackages + rsync "${OUT_RSYNC_OPTS[@]}" ${STAGINGTREE}/${a}/* ${OUTTREE}/${a}/binpackages/ + date -u > ${OUTTREE}/${a}/binpackages/.timestamp +done + + +# we're done so remove the "lockfile" +rm ${STAGINGTREE}/.running diff --git a/sync-binpackages.sh b/sync-binpackages.sh deleted file mode 100755 index fe524d9..0000000 --- a/sync-binpackages.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# Copyright 2011-2023 Gentoo Authors; Distributed under the GPL v2 - -# for testing -ARCHES="sparc" - -# Keep this variable in sync in both sign-binpackages.sh & sync-binpackages.sh -_ARCHES="alpha amd64 arm64 arm hppa ia64 loong m68k mips ppc riscv s390 sh sparc x86" - #alpha amd64 arm64 arm hppa ia64 loong m68k mips ppc riscv s390 sh sparc x86 -ARCHES=${ARCHES:-${_ARCHES}} - -# this is the directory where all packages are signed -# we assume it's on dipper / releng-incoming, but might as well give a full rsync -# specification here -SRCDIR="/release/binpackages" -# append ${a} - -# this is the outgoing directory -DSTDIR="/var/tmp/gmirror-releases/releases" -# append ${a}/binpackages - -RSYNC="/usr/bin/rsync" -RSYNC_OPTS=( - --no-motd - --archive - --ignore-errors - --delete - --delete-after - --timeout=300 - --mkpath -) - -[[ $(whoami) == "gmirror" ]] || exit 111 - -for a in ${ARCHES}; do - [[ -d ${DSTDIR}/${a}/binpackages ]] || mkdir -p ${DSTDIR}/${a}/binpackages - rsync "${RSYNC_OPTS[@]}" ${SRCDIR}/${a}/* ${DSTDIR}/${a}/binpackages/ -done |