path: root/snapshots-create.sh

#!/bin/bash
# Copyright 2011-2015 Gentoo Foundation; Distributed under the GPL v2
# might be earlier copyright, no history available

# 1) Create the tarball
# 2) Sanity check the tarball size and bail out if it appears abnormal.
# 3) create xz tarball
# 4) sign
# 5) delta generation
# 6) create some symlinks
# 7) tidy up
# 8) clean up old deltas

umask 022
source /usr/local/bin/mastermirror/rsync-gen.vars

# locations (if used in multiple files, please move to rsync-gen.vars file)
MASTER="${FINALDIR}" # the master gentoo-x86 copy
TEMP="${BASE}/snapshots-tmp/" # working directory
HISTORY="7" #number in dsays of previous snapshots to keep
DELTA_UPLOAD="${UPLOAD}/deltas/"

[[ -d ${TEMP} ]]			|| mkdir ${TEMP}
[[ -d ${UPLOAD} ]]			|| mkdir ${UPLOAD}
[[ -d ${DELTA_UPLOAD} ]]		|| mkdir ${DELTA_UPLOAD}

write_time_log "---------------------------------------------------------"
write_time_log "START ENTIRE SCRIPT	$(date -u)"

# used to name the file
DELTA_BASE=`/bin/date -d '-2 day' +%Y%m%d`
DELTA_FILENAME="portage-${DELTA_BASE}.tar.bz2"
YESTERDAY=`/bin/date -d yesterday +%Y%m%d`
FILENAME="portage-${YESTERDAY}.tar.bz2"

# GPG info
#SIGNKEYID="D8BA32AA"
#SIGNKEYID="7DDAD20D"
#SIGNKEYID="239C75C4"
SIGNKEYID="96D8BF6D"

if [[ ! $(gpg -k | grep ${SIGNKEYID}) ]]; then
	echo "${SIGNKEYID} not imported! exiting"
	exit 1
fi

if [[ ! -e "${UPLOAD}/${DELTA_FILENAME}" ]]; then
	echo "Previous snapshot does not exist: '${UPLOAD}/${DELTA_FILENAME}'"
	exit 1
fi

# Find fastest BZIP2
for BZIP2_PROG in lbzip2 pbzip2 bzip2 FAIL ; do
	[ -n "$(type $BZIP2_PROG 2>/dev/null)" ] && break
done
if [ $BZIP2_PROG == FAIL ];  then
	echo "Could not find any BZIP2" 1>&2
	exit 1
fi
# Find fastest XZ
for XZ_PROG in pixz xz FAIL ; do
	[ -n "$(type $XZ_PROG 2>/dev/null)" ] && break
done
if [ $XZ_PROG == FAIL ];  then
	echo "Could not find any xz" 1>&2
	exit 1
fi

# working dir
cd ${TEMP}

# 1) Create the tarball
# create the tarball and move it to the right location
write_time_log "START TARBALL		$(date -u)"
if [ ! -f "${FILENAME%.bz2}" ]; then
	rm -rf portage
	rsync -Wqa "$MASTER"/ portage/
	if [ $? -ne 0 ]; then
		echo "rsync run failed!"
		exit 1
	fi
	tar --owner=portage --group=portage -cf ${FILENAME%.bz2} portage \
		--exclude .checksum-test-marker
	rc=$?
	rm -rf portage
	if [ $rc -ne 0 ]; then
		echo "Tar run failed!"
		exit 1
	fi
fi
[ ! -f " ${FILENAME}.umd5sum" ] && md5sum ${FILENAME%.bz2} > ${FILENAME}.umd5sum
[ ! -f "${FILENAME%.bz2}.bz2" ] && $BZIP2_PROG -k9 ${FILENAME%.bz2}
write_time_log "END TARBALL		$(date -u)"
# end 1)

# 2) Sanity check the tarball size and bail out if it appears abnormal.
write_time_log "START SIZE SANITY	$(date -u)"
current_size=$(stat -c '%s' "${FILENAME}")
previous_size=$(stat -c '%s' "${UPLOAD}/${DELTA_FILENAME}")
if [ ${current_size} -lt ${previous_size} ]; then
	size_difference=$(expr ${previous_size} - ${current_size})
	difference_ratio=$(expr ${previous_size} / ${size_difference})
	if [ ${difference_ratio} -lt 5 ]; then
		echo "Snapshot size has decreased by more than 20% in one day!!!"
		echo "${FILENAME} ${current_size} bytes"
		echo "${DELTA_FILENAME} ${previous_size} bytes"
		exit 1
	fi
fi
write_time_log "END SIZE SANITY		$(date -u)"
# end 2)

# 3) create xz tarball
write_time_log "START XZ		$(date -u)"
if [ ! -f "${FILENAME%.*}.xz" ] ; then
	# pixz, pxz, xz all differ in filename generation
	# xz: .tar -> .tar.xz
	# pixz: .tar -> .tpxz
	# pxz: .tar -> .txz
	#
	# To avoid this, be explicit by using IO.
	${XZ_PROG} -9 -e <"${FILENAME%.*}" >"${FILENAME%.*}.xz" || exit $?
fi
write_time_log "END XZ			$(date -u)"
# end 3)

# 4) sign
write_time_log "START SIGN		$(date -u)"
for f in "${FILENAME}" "${FILENAME%.*}".xz ; do
	if [ ! -f "${UPLOAD}${f}".umd5sum ]; then
		cp "${FILENAME}".umd5sum "${UPLOAD}${f}".umd5sum || exit $?
		md5sum "$f" > "$f".md5sum || exit $?
	fi
	if [ ! -f "$f".gpgsig ]; then
		gpg --batch -u "${SIGNKEYID}" --armor --detach-sign \
			--output "$f".gpgsig "$f" || exit $?
	fi
	mv "$f" "$f".md5sum "$f".gpgsig "${UPLOAD}"/ || exit $?
done
write_time_log "END SIGN		$(date -u)"
# end 4)

# 5) delta generation
write_time_log "START DELTA		$(date -u)"
PATCH=snapshot-${DELTA_BASE}-${YESTERDAY}.patch.bz2
if [ ! -f "${PATCH}" ]; then
	${BZIP2_PROG} -dkc ${UPLOAD}/${DELTA_FILENAME} > orig
	/usr/bin/differ -f bdelta orig ${FILENAME%.bz2} ${PATCH%.bz2}
	${BZIP2_PROG} -9 ${PATCH%.bz2}
	md5sum ${PATCH} > ${PATCH}.md5sum
	chmod 644 ${PATCH}{,.md5sum}
	mv ${PATCH}{,.md5sum} ${DELTA_UPLOAD}
	rm orig ${FILENAME%.bz2} "${FILENAME}".umd5sum || exit $?
fi
write_time_log "END DELTA		$(date -u)"
# end 5)

# 6) create some symlinks
write_time_log "START SYMLINK		$(date -u)"
cd ${UPLOAD}
for f in "${FILENAME}" "${FILENAME%.*}".xz ; do
	ext=${f##*.}
	ln -sf "$f" "${UPLOAD}"portage-latest.tar.${ext} || exit $?
	rm -f "${UPLOAD}"portage-latest.tar.${ext}.md5sum || exit $?
	sed "s/${f}\$/portage-latest.tar.${ext}/" "${UPLOAD}"${f}.md5sum > \
		"${UPLOAD}"portage-latest.tar.${ext}.md5sum || exit $?
	ln -sf "${f}".gpgsig "${UPLOAD}"portage-latest.tar.${ext}.gpgsig || exit $?
done
write_time_log "END SYMLINK		$(date -u)"
# end 6)

# 7) tidy up
write_time_log "START CLEANUP		$(date -u)"
/usr/bin/find ${UPLOAD} -maxdepth 1 -mtime +${HISTORY} -type f | /usr/bin/xargs /bin/rm -f
write_time_log "END CLEANUP		$(date -u)"
# end 7)

# 8) clean up old deltas
write_time_log "START CLEANUP DELTA	$(date -u)"
/usr/local/bin/mastermirror/clean-old-deltas.py "${DELTA_UPLOAD}" "${YESTERDAY}" $(stat -c '%s' "${UPLOAD}/${FILENAME}") > /dev/null
write_time_log "END CLEANUP DELTA	$(date -u)"
# end 8)
write_time_log "END ENTIRE SCRIPT	$(date -u)"