diff options
| -rwxr-xr-x | misc/emerge-delta-webrsync | 512 |
1 files changed, 512 insertions, 0 deletions
diff --git a/misc/emerge-delta-webrsync b/misc/emerge-delta-webrsync new file mode 100755 index 000000000..6e1f5cff0 --- /dev/null +++ b/misc/emerge-delta-webrsync @@ -0,0 +1,512 @@ +#!/bin/bash +# Copyright 1999-2004 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# Author: Brian Harring <ferringb@gentoo.org>, karltk@gentoo.org originally. +# Rewritten from the old, Perl-based emerge-webrsync script + +#------------------- +#initialization +#------------------ + +f=$(python -c 'import portage, sys; sys.stdout.write("|".join([portage.settings[x] for x in ("PORTAGE_NICENESS", "FEATURES", "GENTOO_MIRRORS", "PORTDIR", "FETCHCOMMAND", "USERLAND", "DISTDIR", "PORTAGE_TMPDIR")]))') + +IFS='|' +PORTAGE_NICENESS="${f%%|*}"; f="${f#*|}" +FEATURES="${f%%|*}" ; f="${f#*|}" +GENTOO_MIRRORS="${f%%|*}" ; f="${f#*|}" +PORTDIR="${f%%|*}" ; f="${f#*|}" +FETCHCOMMAND="${f%%|*}" ; f="${f#*|}" +USERLAND="${f%%|*}" ; f="${f#*|}" +DISTDIR="${f%%|*}" ; f="${f#*|}" +TMPDIR="${f%%|*}/snapshots" +unset IFS + +source /usr/lib/portage/bin/isolated-functions.sh || exit 1 + +if [ -z "$NICENESS_PULLED" ]; then + if [ -n "${PORTAGE_NICENESS}" ]; then + export NICENESS_PULLED=asdf + exec nice -n "${PORTAGE_NICENESS}" "$0" "$@" + echo "failed setting PORTAGE_NICENESS to '$PORTAGE_NICENESS', disabling" + fi +fi + +STATE_DIR="/var/delta-webrsync/" + +# hack. bug 92224 +if [ "${FETCHCOMMAND/getdelta.sh}" != "${FETCHCOMMAND}" ]; then + # evil evil evil evil + eval "$(grep FETCHCOMMAND /etc/make.globals)" +fi + +unset f +unset IFS + +MUST_SYNC='1' +unset PUKE_HELP wgetops +for x in $*; do + if [[ $x == "-u" ]]; then + MUST_SYNC='' + elif [[ $x == "-k" ]]; then + KEEP_OLDIES='asdf' + elif [[ $x == "-h" ]]; then + PUKE_HELP=1 + elif [[ $x == "-v" ]]; then + wgetops= + else + PUKE_HELP=1 + echo "$x isn't a valid arg. bailing." + fi + if [[ -n $PUKE_HELP ]]; then + echo "-u for upgrade; sync only if new snapshots are found" + echo "-k for keep; keep old tree snapshots around" + exit -1 + fi +done + +if [[ ! -d $STATE_DIR ]]; then + echo "$STATE_DIR doesn't exist. don't have the ability to compensate for compressor differences without it!" + exit -2 +fi + +if [[ ! -d $DISTDIR ]] ; then + mkdir -p $DISTDIR +fi +if [[ ! -d $TMPDIR ]]; then + mkdir -p $TMPDIR +fi + +cd "$DISTDIR" + +found=0 +if [ "${wgetops-UNSET}" == "unset" ]; then + #this sucks. probably better to do 1> /dev/null + #that said, waiting on the refactoring. + if [ "${FETCHCOMMAND/wget}" != "${FETCHCOMMAND}" ]; then + wgetops="-q" + elif [ "${FETCHCOMMAND/curl}" != "${FETCHCOMMAND}" ]; then + wgetops="-s -f" + fi +fi + +if type -p md5sum > /dev/null; then + md5_com='md5sum -c "${MD5_LOC}" &> /dev/null' +elif type -p md5 > /dev/null; then + md5_com='[ "$(md5 -q ${FILE})" == "$(cut -d \ -f 1 ${MD5_LOC})" ]' +else + echo "warning, unable to do md5 verification of the snapshot!" + echo "no suitable md5/md5sum binary was found!" + md5_com='true' +fi + +#--------------- +#funcs +#--------------- + +cleanse_state_dir() { + [[ ${STATE_DIR:-/} != '/' ]] && rm ${STATE_DIR}/* &> /dev/null; +} + +full_version_attempt() { + local FILE file_exists + echo "Fetching most recent snapshot" + declare -i attempts=-1 + while (( $attempts < 40 )) ; do + unset file_exists + attempts=$(( attempts + 1 )) + + #this too, sucks. it works in the interim though. + if [ "$USERLAND" == "BSD" ] || [ "$USERLAND" == "Darwin" ] ; then + daysbefore=$(expr $(date +"%s") - 86400 \* $attempts) + day=$(date -r $daysbefore +"%d") + month=$(date -r $daysbefore +"%m") + year=$(date -r $daysbefore +"%Y") + else + day=$(date -d "-$attempts day" +"%d") + month=$(date -d "-$attempts day" +"%m") + year=$(date -d "-$attempts day" +"%Y") + fi + + FILE="portage-${year}${month}${day}.tar.bz2" + + echo "Attempting to fetch file dated: ${year}${month}${day}" + + got_md5=0 + + if [ ! -e "${FILE}.md5sum" ]; then + fetch_from_mirrors "/snapshots/${FILE}.md5sum" "${FILE}.md5sum" + got_md5=$? + else + file_exists='asdf' + got_md5=0 + fi + + if [[ $got_md5 != 0 ]]; then + echo " --- No md5sum present on the mirror. (Not yet available.)" + continue + elif [ -s "${FILE}" ]; then + if verify_md5_file "$FILE"; then + echo " === snapshot $FILE is correct, using it" + if [[ -n $MUST_SYNC ]] || [[ -z file_exists ]]; then + sync_local "${FILE}" + echo + echo " === Snapshot has been sync'd" + echo + else + echo + echo "skipped sync" + echo + fi + exit 0 + else + echo "md5 on ${FILE} failed, removing it and starting anew" + rm "$FILE" &> /dev/null + fi + fi + + if fetch_from_mirrors "/snapshots/${FILE}" "${FILE}"; then + if ! verify_md5_file "$FILE"; then + echo "md5 failed on $FILE" + rm ${FILE} &> /dev/null + continue + else + sync_local "${FILE}" + cleanse_state_dir + echo + echo " *** Completed websync, please now perform a normal rsync if possible." + echo " Update is current as of YYYYMMDD: ${year}${month}${day}" + echo + exit 0 + fi + fi + + done + exit 1 +} + + +sync_local() { + local FILE flags + FILE="$1" + if [ "${FILE/\/}" == "${FILE}" ]; then + FILE="${DISTDIR}/${FILE}"; + fi + + echo Syncing local tree... + if type -p tarsync &> /dev/null; then + echo "apparently you have tarsync installed. using it." + if ! tarsync "${FILE}" "${PORTDIR}" -v -s 1 -o portage -g portage -e /distfiles -e /packages -e /local; then + echo "ok, tarsync failed. that's teh suck :/" + exit 6 + fi + else + cd ${TMPDIR} || die "couldn't cd to tmpdir, $TMPDIR!?" + flags="xf" + if [ "${FILE%.bz2}" != "${FILE}" ]; then + flags="jxf" + fi + if ! tar ${flags} "$FILE"; then + echo "Tar failed to extract the image. Please review the output." + echo "Executed command: tar jxf $FILE" + exit 1 + fi + # Make sure user and group file ownership is root + chown -R 0:0 portage + cd portage + rsync -av --progress --stats --delete --delete-after \ + --exclude='/distfiles' --exclude='/packages' \ + --exclude='/local' . ${PORTDIR%%/} + cd .. + echo "cleaning up" + rm -rf portage + fi + if has metadata-transfer ${FEATURES} ; then + echo "transferring metadata/cache" + emerge --metadata + fi + local post_sync=/etc/portage/bin/post_sync + [[ -x "${post_sync}" ]] && ${post_sync} +} + +fetch_from_mirrors() { + local i URI FILE MIRRORS + if [[ "$#" == 3 ]]; then + MIRRORS="${3}" + else + MIRRORS=$GENTOO_MIRRORS + fi + FILE="$2" + for i in $MIRRORS ; do + URI="${i}/${1}" + if (eval "$FETCHCOMMAND $wgetops") && [ -s "${FILE}" ]; then + return 0 + else + rm "${FILE}" &> /dev/null + fi + done + return 1 +} + +verify_md5_file() { + local FILE MD5_LOC CUR + FILE="$1" + if [[ $# == 2 ]]; then + MD5_LOC="$2" + else + MD5_LOC="$(pwd)/$1.md5sum" + fi + if [ "${FILE/*\/}" != "$1" ]; then + CUR="$(pwd)" + cd "$(dirname ${FILE})" + FILE="$(basename ${FILE})" + fi + if eval "$md5_com"; then + [ -n "${CUR}" ] && cd "${CUR}" + return 0 + else + [ -n "${CUR}" ] && cd "${CUR}" + return 1 + fi +} + + +#-------------------- +#inline actual script +#-------------------- + +if ! type -p patcher &> /dev/null; then + echo "!!!" + echo "!!! cannot find patcher, did you emerge dev-util/diffball?" + echo "!!! lack of patcher == have to do full fetch" + echo "!!!" + sleep 10 + full_version_attempt +fi + +echo "Looking for available base versions for a delta" + +#note we're already in distdir + +unset base_version +# portage-snapshots in reverse order. +# icky. +unset dfile +potentials="$(ls -1 portage-2[[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]].tar.bz2 ${STATE_DIR}/portage-2[[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]].tar.bz2 2> /dev/null | sed -e 's:^.*/::' | sort -r)" +for basef in ${potentials}; do + chksum='' + found="dar" + if [ -e "${STATE_DIR}/${basef}.md5sum" ]; then + chksum="${STATE_DIR}/${basef}.md5sum" + elif [ -e "${basef}.md5sum" ]; then + chksum="${DISTDIR}/${basef}.md5sum" + else + echo "attempting to get md5sum for $basef" + if ! fetch_from_mirrors "/snapshots/${basef}.md5sum" "${basef}.md5sum"; then + echo "can't get md5 for ${basef}" + continue + fi + chksum="${basef}.md5sum" + fi + if [ -e "${basef}" ]; then + dfile="${DISTDIR}/${basef}" + else + dfile="${STATE_DIR}/${basef}" + fi + if ! verify_md5_file "${dfile}" "${chksum}"; then + echo "found a stale snapshot. cleansing" + rm "${dfile}" &> /dev/null + rm "${chksum}.md5sum" &> /dev/null + dar="" + else + base_version="${basef}" + break + fi +done + +#by this point, we either have a base_version, or we don't. +if [[ -z ${base_version} ]]; then + echo "no base found. resorting to pulling a full version" + full_version_attempt +fi + +#we have a md5 verified base. now we get the patch. + +base_date="${base_version%.tar.bz2}" +base_date="${base_date#portage-}" +# we now have yyyymmdd + +unset patch_dates +if [ "$USERLAND" == "BSD" ] || [ "$USERLAND" == "Darwin" ] ; then + daysbefore=$(expr $(date +"%s") - 86400 \* $attempts) + day=$(date -r $daysbefore +"%d") + month=$(date -r $daysbefore +"%m") + year=$(date -r $daysbefore +"%Y") +else + day=$(date -d "-$attempts day" +"%d") + month=$(date -d "-$attempts day" +"%m") + year=$(date -d "-$attempts day" +"%Y") +fi + +#todays_date="${year}${month}${day}" + +next_date() { + local year day month + year="${1:0:4}" + month="${1:4:2}" + day="${1:6:2}" +# if [[ "${USERLAND}" == "BSD" ]] || [[ "${USERLAND}" == "Darwin" ]]; then +# else + date -d "$year/$month/$day +1 day" +"%Y%m%d" +# fi +} + +patches='' +echo "fetching patches" +fetched='asdf' +while [[ -n ${fetched} ]]; do + next_day=$(next_date ${base_date}) + # if we can't get a *single* patch or md5, even one missing, do full. + p="snapshot-${base_date}-${next_day}.patch.bz2" + if [[ ! -e ${p}.md5sum ]] && ! fetch_from_mirrors "/snapshots/deltas/${p}.md5sum" "${p}.md5sum"; then + echo "failed fetching ${p}.md5sum" + fetched='' + break + fi + fetch="yes" + if [[ -e ${p} ]]; then + if ! verify_md5_file "${p}"; then + rm ${p} &> /dev/null + else + fetch="" + fi + fi + if [[ -n $fetch ]]; then + if ! fetch_from_mirrors "/snapshots/deltas/${p}" "${p}"; then + echo "failed fetching ${p}" + fetched='' + fi + fi + if [[ -z ${fetched} ]]; then + break + fi + if ! verify_md5_file "${p}"; then + echo "md5 failed on ${p}" + fetched='' + break + fi + patches="${patches} ${p}" + base_date="${next_day}" +done +final_date=${base_date} + +if [[ -z $patches ]]; then + echo "no patches found? up to date?" + if [[ -n $MUST_SYNC ]]; then + echo "syncing with existing file" + sync_local "${dfile}" + else + : + fi + exit $? +fi + +unset got_umd5 +#grab the md5 for later usage. +if [[ ! -e portage-${final_date}.tar.bz2.md5sum ]] && ! fetch_from_mirrors "/snapshots/portage-${final_date}.tar.bz2.md5sum" "portage-${final_date}.tar.bz2.md5sum"; then + echo "warning... couldn't grab the md5sum for ${final_date}. which is odd" + echo "thus, bailing (sorry)" + exit 5 +else + if [[ ! -e portage-${final_date}.tar.bz2.umd5sum ]] && ! fetch_from_mirrors "/snapshots/portage-${final_date}.tar.bz2.umd5sum" "portage-${final_date}.tar.bz2.umd5sum"; then + if ! fetch_from_mirrors "/snapshots/portage-${final_date}.tar.bz2.umd5sum" "portage-${final_date}.tar.bz2.umd5sum"; then + echo "couldn't grab umd5sum (uncompressed md5sum) for ${final_date}." + echo "can't compensate for bzip2 version differences iow." + else + got_umd5=1 + fi + else + got_umd5=1 + fi +fi + +# generate tmp dir. +TEMPDIR=$(mktemp -d /tmp/delta-webrsync-XXXXXX) +# got our patches. +if ! patcher -v "${dfile}" ${patches} "${TEMPDIR}/portage-${final_date}.tar"; then + echo "reconstruction failed (contact the author with the error from the reconstructor please)" + rm "${TEMPDIR}/portage-${final_date}.tar" + rmdir ${TEMPDIR} + full_version_attempt +fi +verified=0 +if [[ -n $got_umd5 ]]; then + echo "verifying uncompressed md5" + if ! verify_md5_file "${TEMPDIR}/portage-${final_date}.tar" "${DISTDIR}/portage-${final_date}.tar.bz2.umd5sum"; then + echo "uncompressed verification failed. This means either you found a bug in diffball, or something odd is going on" + echo "with upstream patch generation" + echo "trying md5sum next, which probably will fail." + else + verified="1" + fi +fi + +unset need_last_sync +if [ "$verified" == "1" ]; then + echo "recompressing. (backgrounding)" + need_last_sync="dar" + bzip2 -vk9 "${TEMPDIR}/portage-${final_date}.tar" & + + echo "beginning update to the tree" + sync_local "${TEMPDIR}/portage-${final_date}.tar" + echo "doing final md5 stuff" + wait + # bzip2 is finished now. + rm "${TEMPDIR}/portage-${final_date}.tar" +else + echo "recompressing." + bzip2 -v9 "${TEMPDIR}/portage-${final_date}.tar.bz2" +fi + +echo "verifying generated tarball" + +if ! verify_md5_file "${TEMPDIR}/portage-${final_date}.tar.bz2" "${DISTDIR}/portage-${final_date}.tar.bz2.md5sum"; then + if [[ -z $verified ]]; then + echo "couldn't verify the generated tarball. bug, most likely." + exit 5 + fi + # hokay. md5 doesn't agree with umd5. bzip2 issue in effect. + echo "compressed md5 differs, but uncompressed md5 says it right. bzip2 version incompatability in other words" + echo "saving the md5" + if type -p md5sum &> /dev/null; then + md5sum ${TEMPDIR}/portage-${final_date}.tar.bz2 | sed -e "s:${TEMPDIR}/\?::" > \ + ${STATE_DIR}/portage-${final_date}.tar.bz2.md5sum + elif type -p md5 &> /dev/null; then + echo "$(md5 -q ${TEMPDIR}/portage-${final_date}.tar.bz2) portage-${final_date}.tar.bz2" > \ + ${STATE_DIR}/portage-${final_date}.tar.bz2.md5sum + else + echo "couldn't find either md5 or md5sum. something is screwed... (bailing, sorry)" + exit 7 + fi + mv "${DISTDIR}/portage-${final_date}.tar.bz2.umd5sum" "${TEMPDIR}/portage-${final_date}.tar.bz2" ${STATE_DIR}/ + rmdir ${TEMPDIR} + dfile="${STATE_DIR}/portage-${final_date}.tar.bz2" +else + dfile="${DISTDIR}/portage-${final_date}.tar.bz2" + mv "${TEMPDIR}/portage-${final_date}.tar.bz2" ${DISTDIR}/ +fi + +if [ -z "${need_last_sync}" ]; then + echo "beginning update to the tree" + sync_local "${dfile}" +fi + +if [[ -z $KEEP_OLDIES ]]; then + echo "cleansing" + for x in $potentials; do + echo "removing ${x}" + rm "${DISTDIR}/${x}" "${DISTDIR}/${x}.md5sum" &> /dev/null + rm "${STATE_DIR}/${x}" "${STATE_DIR}/${x}.md5sum" "${STATE_DIR}/${x}.umd5sum" &> /dev/null + done +fi +echo "done." + |