snapshots-create.sh: refactor tar options and include full tarformat rationale

Signed-off-by: Robin H. Johnson <robbat2@gentoo.org>

1 files changed, 45 insertions, 24 deletions

diff --git a/snapshots-create.sh b/snapshots-create.sh
index b230ca6..5ea56ec 100755
--- a/snapshots-create.sh
+++ b/snapshots-create.sh
@@ -104,49 +104,70 @@ EXCLUSION_LIST="$(mktemp -p ${TEMP} snapshot-exclude.XXXXXXXXXX)"
 write_time_log "END TARBALL(prep)		$(date -u)"
 
 # 1b) Create the tarball
-# create the tarball and move it to the right location
-write_time_log "START TARBALL(tar,old)		$(date -u)"
-if [ ! -f "${FILENAME%.bz2}" ]; then
-	TAR_OPTIONS=(
+COMMON_TAR_OPTIONS=(
 	# Force a small block size
-		--blocking-factor=1
-		--record-size=512
-	# GNU tar format saves approximately 1K per file in the tarball over POSIX
-	# format. Multiply ~170k files, and the savings are large.
-		--format=gnu
+	--blocking-factor=1
+	--record-size=512
+	# Tar format rationale:
+	# ---------------------
+	# Longest directory prefix, 94 chars:
+	# gentoo-YYYYMMDD/profiles/default/linux/powerpc/ppc64/17.0/64bit-userland/little-endian/systemd/
+	# Longest path, 140 chars:
+	# gentoo-YYYYMMDD/gnome-extra/gnome-shell-extension-applications-overview-tooltip/gnome-shell-extension-applications-overview-tooltip-6.ebuild
+	# Longest filename, 91 chars:
+	# spirv-tools-2019.10_pre20191027-Respect-CMAKE_INSTALL_LIBDIR-in-installed-CMake-files.patch
+	#
+	# These length require that a tar format with unlimited length is used, the
+	# split-255 layout of ustar is not gaurenteed to be long enough.
+	#
+	# The tar formats with unlimited length are: gnutar, posix
+	#
+	# The posix tar format has additional 1K overhead per entry, and the Gentoo
+	# repo has ~160k entries (files & dirs), which adds up to 160M extra in the
+	# tarball.
+	# The differ/bdelta toolchain is also confirmed to work on posix tar
+	# format.
+	--format=gnu
 	# Sorting by name produces consistent ordering and helps compression of
 	# related content. Custom ordering might further improve ordering in future
 	# (eg all metadata.xml first)
-		--sort=name
+	--sort=name
 	# Force ownership of content:
-		--owner=portage
-		--group=portage
+	--owner=portage
+	--group=portage
 	# Excluded content:
-		--no-wildcards
-		--exclude-from "${EXCLUSION_LIST}"
+	--no-wildcards
+	--exclude-from "${EXCLUSION_LIST}"
 	# Do not capture any xattr/acl info at all.
-		--no-acls
-		--no-xattrs
-		--no-selinux
+	--no-acls
+	--no-xattrs
+	--no-selinux
 	# Include a volume ID for tracing
-	# volume header is not supported by:
+	# volume header is NOT supported by:
 	# - Docker https://bugs.gentoo.org/631644
 	# - tarsync https://bugs.gentoo.org/631616
-	#	-V "${FILENAME%.bz2}"
+	#-V "${FILENAME%.bz2}"
 	# do everything relative to the destination
-		-C "${MASTER}"
+	-C "${MASTER}"
+)
+# create the tarball and move it to the right location
+write_time_log "START TARBALL(tar,old)		$(date -u)"
+if [ ! -f "${FILENAME%.bz2}" ]; then
+	OLD_TARBALL_OPTIONS=(
 		# The . needs to match the file argument
 		--transform='s,^\.,portage,g'
-	# The operation, destination, source arguments
+		# The operation, destination, source arguments
 		--create
 		--file ${FILENAME%.bz2}
 		.
 	)
-
-	tar "${TAR_OPTIONS[@]}"
+	tar \
+		"${COMMON_TAR_OPTIONS[@]}" \
+		"${OLD_TARBALL_OPTIONS[@]}"
 	rc=$?
 	if [ $rc -ne 0 ]; then
-		echo "Tar run failed!"
+		echo "TARBALL(tar,old) failed!"
+		echo "TARBALL(tar,old) failed!" 1>&2
 		exit 1
 	fi
 fi