diff options
-rwxr-xr-x | scripts/auto-bootstraps/analyse_result.py | 178 | ||||
-rwxr-xr-x | scripts/auto-bootstraps/dobootstrap | 167 | ||||
-rwxr-xr-x | scripts/auto-bootstraps/process_uploads.sh | 60 | ||||
-rwxr-xr-x | scripts/auto-bootstraps/update_distfiles.py | 29 |
4 files changed, 434 insertions, 0 deletions
diff --git a/scripts/auto-bootstraps/analyse_result.py b/scripts/auto-bootstraps/analyse_result.py new file mode 100755 index 0000000000..885c7fc9e7 --- /dev/null +++ b/scripts/auto-bootstraps/analyse_result.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 + +import os +import glob +import re +import time +import html + +resultsdir='./results' + +def find_last_stage(d): + """ + Returns the last stage worked on. + Bootstraps define explicitly stages 1, 2 and 3, we define some more + on top of those as follows: + 0 - bootstrap didn't even start (?!?) or unknown status + 1 - stage 1 failed + 2 - stage 2 failed + 3 - stage 3 failed + 4 - emerge -e world failed + 5 - finished successfully + """ + + def stage_success(stagelog): + with open(stagelog, 'rb') as f: + line = f.readlines()[-1] + res = re.match(r'^\* stage[123] successfully finished', + line.decode('utf-8', 'ignore')) + return res is not None + + if not os.path.exists(os.path.join(d, '.stage1-finished')): + log = os.path.join(d, 'stage1.log') + if not os.path.exists(log): + return 0 # nothing exists, assume not started + if not stage_success(log): + return 1 + + if not os.path.exists(os.path.join(d, '.stage2-finished')): + log = os.path.join(d, 'stage2.log') + if not os.path.exists(log) or not stage_success(log): + return 2 # stage1 was success, so 2 must have failed + + if not os.path.exists(os.path.join(d, '.stage3-finished')): + log = os.path.join(d, 'stage3.log') + if not os.path.exists(log) or not stage_success(log): + return 3 # stage2 was success, so 3 must have failed + + # if stage 3 was success, we went onto emerge -e system, if that + # failed, portage would have left a build.log behind + logs = glob.glob(d + "/portage/*/*/temp/build.log") + if len(logs) > 0: + return 4 + + # ok, so it must have been all good then + return 5 + +def get_err_reason(arch, dte, err): + rdir = os.path.join(resultsdir, arch, '%d' % dte) + + if err == 0: + return "bootstrap failed to start" + if err >= 1 and err <= 3: + stagelog = os.path.join(rdir, 'stage%d.log' % err) + if os.path.exists(stagelog): + line = None + with open(stagelog, 'rb') as f: + errexp = re.compile(r'^( \* (ERROR:|Fetch failed for)|emerge: there are no) ') + for line in f: + res = errexp.match(line.decode('utf-8', 'ignore')) + if res: + break + if not line: + return '<a href="%s/stage%d.log">stage %d</a> failed' % \ + (os.path.join(arch, '%d' % dte), err, err) + return '<a href="%s/stage%d.log">stage %d</a> failed<br />%s' % \ + (os.path.join(arch, '%d' % dte), err, err, \ + html.escape(line.decode('utf-8', 'ignore'))) + else: + return 'stage %d did not start' % err + if err == 4: + msg = "'emerge -e system' failed while emerging" + logs = glob.glob(rdir + "/portage/*/*/temp/build.log") + for log in logs: + cat, pkg = log.split('/')[-4:-2] + msg = msg + ' <a href="%s/temp/build.log">%s/%s</a>' % \ + (os.path.join(arch, '%d' % dte, "portage", cat, pkg), \ + cat, pkg) + return msg + +def analyse_arch(d): + last_fail = None + last_succ = None + fail_state = None + with os.scandir(d) as it: + for f in sorted(it, key=lambda x: (x.is_dir(), x.name), reverse=True): + if not f.is_dir(follow_symlinks=False): + continue + date = int(f.name) + res = find_last_stage(os.path.join(d, f.name)) + if res == 5: + if not last_succ: + last_succ = date + elif not last_fail: + last_fail = date + fail_state = res + if last_succ and last_fail: + break + + return (last_fail, fail_state, last_succ) + +archs = {} +with os.scandir(resultsdir) as it: + for f in sorted(it, key=lambda x: (x.is_dir(), x.name)): + if not f.is_dir(follow_symlinks=False): + continue + arch = f.name + fail, state, suc = analyse_arch(os.path.join(resultsdir, arch)) + archs[arch] = (fail, state, suc) + if not suc: + color = '\033[1;31m' # red + elif fail and suc < fail: + color = '\033[1;33m' # yellow + else: + color = '\033[1;32m' # green + endc = '\033[0m' + print("%s%24s: suc %8s fail %8s%s" % (color, arch, suc, fail, endc)) + +# generate html edition +with open(os.path.join(resultsdir, 'index.html'), "w") as h: + h.write("<html>") + h.write("<head><title>Gentoo Prefix bootstrap results</title></head>") + h.write("<body>") + h.write("<h2>Gentoo Prefix bootstraps</h2>") + h.write('<table border="1px">') + h.write("<th>architecture</th>") + h.write("<th>last successful run</th><th>last failed run</th>") + h.write("<th>failure</th>") + for arch in archs: + fail, errcode, suc = archs[arch] + if not suc: + state = 'red' + elif fail and suc < fail: + state = 'orange' + else: + state = 'limegreen' + + h.write('<tr>') + + h.write('<td bgcolor="%s" nowrap="nowrap">' % state) + h.write(arch) + h.write("</td>") + + h.write("<td>") + if suc: + h.write('<a href="%s/%s">%s</a>' % (arch, suc, suc)) + else: + h.write('<i>never</i>') + h.write("</td>") + + h.write("<td>") + if fail: + h.write('<a href="%s/%s">%s</a>' % (arch, fail, fail)) + else: + h.write('<i>never</i>') + h.write("</td>") + + h.write("<td>") + if fail and (not suc or fail > suc): + h.write(get_err_reason(arch, fail, errcode)) + h.write("</td>") + + h.write("</tr>") + h.write("</table>") + now = time.strftime('%Y-%m-%d %H:%M', time.gmtime()) + h.write("<p><i>generated: %s</i></p>" % now) + h.write("<p>See also <a href='https://dev.azure.com/12719821/12719821/_build?definitionId=6'>awesomebytes</a></p>") + h.write("</body>") + h.write("</html>") diff --git a/scripts/auto-bootstraps/dobootstrap b/scripts/auto-bootstraps/dobootstrap new file mode 100755 index 0000000000..521f644acf --- /dev/null +++ b/scripts/auto-bootstraps/dobootstrap @@ -0,0 +1,167 @@ +#!/usr/bin/env bash + +BOOTSTRAP="${BASH_SOURCE[0]%/*}/bootstrap-prefix.sh" +BOOTURL="http://rsync.prefix.bitzolder.nl/scripts/bootstrap-prefix.sh" +UPLOAD="rsync1.prefix.bitzolder.nl::gentoo-portage-bootstraps" + +do_fetch() { + local FETCHCOMMAND + # Try to find a download manager, we only deal with wget, + # curl, FreeBSD's fetch and ftp. + if [[ x$(type -t wget) == "xfile" ]] ; then + FETCH_COMMAND="wget -O -" + [[ $(wget -h) == *"--no-check-certificate"* ]] && \ + FETCH_COMMAND+=" --no-check-certificate" + elif [[ x$(type -t curl) == "xfile" ]] ; then + FETCH_COMMAND="curl -f -L" + else + echo "could not download ${1##*/}" + exit 1 + fi + + ${FETCH_COMMAND} "${*}" || exit 1 +} + +do_prepare() { + local bitw=$1 + local dte=$2 + local bootstrap + + if [[ -n ${RESUME} && -n ${bitw} && -n ${dte} ]] ; then + bootstrap=bootstrap${bitw}-${dte}/bootstrap-prefix.sh + elif [[ -n ${DOLOCAL} ]] ; then + bootstrap=${BOOTSTRAP} + else + bootstrap=dobootstrap-do_prepare-$$ + do_fetch ${BOOTURL} > ${bootstrap} + fi + + local chost=$(${BASH} ${bootstrap} chost.guess x) + case ${chost} in + *86-*) + if [[ ${bitw} == 64 ]] ; then + chost=x86_64-${chost#*-} + else + bitw=32 + chost=i386-${chost#*-} + fi + ;; + x86_64-*) + if [[ ${bitw} == 32 ]] ; then + chost=i386-${chost#*-} + else + bitw=64 + chost=x86_64-${chost#*-} + fi + ;; + powerpc-*) + bitw=32 + ;; + sparc-*) + if [[ ${bitw} == 64 ]] ; then + chost=sparcv9-${chost#*-} + else + bitw=32 + chost=sparc-${chost#*-} + fi + ;; + sparcv9-*|sparc64-*) + if [[ ${bitw} == 32 ]] ; then + chost=sparc-${chost#*-} + else + bitw=64 + chost=sparcv9-${chost#*-} + fi + ;; + *) + echo "unhandled CHOST: ${chost}" + rm -f dobootstrap-do_prepare-$$ + exit 1 + ;; + esac + + [[ -z ${dte} ]] && dte=$(date "+%Y%m%d") + EPREFIX=${PWD}/bootstrap${bitw}-${dte} + [[ -n ${OVERRIDE_EPREFIX} ]] && EPREFIX=${OVERRIDE_EPREFIX} + + local bootstrapscript=$(realpath ${BASH_SOURCE[0]} 2>/dev/null) + if [[ -z ${bootstrapscript} ]] ; then + local b=${BASH_SOURCE[0]} + cd "${b%/*}" + bootstrapscript=$(pwd -P)/${b##*/} + fi + echo "EPREFIX=${EPREFIX}" + mkdir -p "${EPREFIX}" + if [[ ${bootstrap} == dobootstrap-do_prepare-$$ ]] ; then + mv "${bootstrap}" "${EPREFIX}"/bootstrap-prefix.sh + elif [[ ${bootstrap} != "${EPREFIX}"/bootstrap-prefix.sh ]] ; then + cp "${bootstrap}" "${EPREFIX}"/bootstrap-prefix.sh + fi + cd "${EPREFIX}" || exit 1 + + # optional program to keep the machine from sleeping + # macOS/BSD: caffeinate + keepalive=$(type -P caffeinate) + [[ -x ${keepalive} ]] && keepalive+=" -i -m -s" || keepalive= + + env -i \ + HOME=${EPREFIX} \ + SHELL=/bin/bash \ + TERM=${TERM} \ + USER=${USER} \ + CHOST=${chost} \ + GENTOO_MIRRORS="http://distfileslocal/" \ + EPREFIX=${EPREFIX} \ + ${DOLOCAL+DOLOCAL=1} \ + ${RESUME+RESUME=1} \ + ${LATEST_TREE_YES+LATEST_TREE_YES=1} \ + ${TREE_FROM_SRC+TREE_FROM_SRC=}${TREE_FROM_SRC} \ + ${keepalive} /bin/bash -l -c "${BASH} ${bootstrapscript} bootstrap" + + if [[ -n ${DOPUBLISH} ]] ; then + rsync -q /dev/null ${UPLOAD}/${HOSTNAME}-$$/ + rsync -q /dev/null ${UPLOAD}/${HOSTNAME}-$$/${chost}/ + rsync -rltv \ + --exclude=work/ \ + --exclude=homedir/ \ + --exclude=files \ + --exclude=distdir/ \ + --exclude=image/ \ + {stage,.stage}* \ + bootstrap-prefix.sh \ + startprefix \ + usr/portage/distfiles \ + var/tmp/portage \ + var/log/emerge.log \ + ${UPLOAD}/${HOSTNAME}-$$/${chost}/${dte}/ + rsync -q /dev/null ${UPLOAD}/${HOSTNAME}-$$/${chost}/${dte}/push-complete/ + fi +} + +do_bootstrap() { + chmod 755 bootstrap-prefix.sh || exit 1 + ${BASH} ./bootstrap-prefix.sh ${EPREFIX} noninteractive +} + +case $1 in + bootstrap) + do_bootstrap + ;; + local) + export DOLOCAL=1 + do_prepare $2 + ;; + resume) + export RESUME=1 + do_prepare "$2" ${3:-${BOOTSTRAP_DATE}} + ;; + *) + if [[ ${0} == /net/* ]] ; then + echo "internal host, activating local and DOPUBLISH" + export DOLOCAL=1 + export DOPUBLISH=1 + fi + do_prepare $1 + ;; +esac + diff --git a/scripts/auto-bootstraps/process_uploads.sh b/scripts/auto-bootstraps/process_uploads.sh new file mode 100755 index 0000000000..52bb09ed7f --- /dev/null +++ b/scripts/auto-bootstraps/process_uploads.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +UPLOADDIR="./uploads" +RESULTSDIR="./results" + +didsomething= +for d in ${UPLOADDIR}/* ; do + if [[ ! -d "${d}" ]] ; then + rm -f "${d}" + continue + fi + + # structure: randomid/chost/date + # chost/date should be the only thing in randomid/ check this + set -- "${d}"/*/* + if [[ $# -ne 1 ]] || [[ ! -d "$1" ]] ; then + rm -Rf "${d}" + continue + fi + + dir=${1#${d}/} + # skip this thing from auto-processing if it is new platform + [[ -d ${RESULTSDIR}/${dir%/*} ]] || continue + # skip this thing if it already exists + [[ -d ${RESULTSDIR}/${dir} ]] && continue + # skip this thing if it isn't complete yet + [[ -d ${d}/${dir}/push-complete ]] || continue + + # only copy over what we expect, so we leave any uploaded cruft + # behind + mkdir "${RESULTSDIR}/${dir}" + for f in \ + stage{1,2,3}.log \ + .stage{1,2,3}-finished \ + bootstrap-prefix.sh \ + emerge.log \ + startprefix \ + distfiles ; + do + [[ -e "${d}/${dir}/${f}" ]] && \ + mv "${d}/${dir}/${f}" "${RESULTSDIR}/${dir}"/ + done + if [[ -e "${d}/${dir}/portage" ]] ; then + for pkg in "${d}/${dir}/portage"/*/* ; do + w=${pkg#${d}/} + mkdir -p "${RESULTSDIR}/${w}" + [[ -e "${pkg}"/build-info ]] && \ + mv "${pkg}"/build-info "${RESULTSDIR}/${w}"/ + [[ -e "${pkg}"/temp ]] && \ + mv "${pkg}"/temp "${RESULTSDIR}/${w}"/ + done + fi + chmod -R o+rX,go-w "${RESULTSDIR}/${dir}" + rm -Rf "${d}" + + [[ -e "${RESULTSDIR}/${dir}"/distfiles ]] && \ + ./update_distfiles.py "${RESULTSDIR}/${dir}"/distfiles > /dev/null + didsomething=1 +done +[[ -n ${didsomething} ]] && ./analyse_result.py > /dev/null diff --git a/scripts/auto-bootstraps/update_distfiles.py b/scripts/auto-bootstraps/update_distfiles.py new file mode 100755 index 0000000000..8f44f7fa20 --- /dev/null +++ b/scripts/auto-bootstraps/update_distfiles.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3.6 + +import hashlib +import os +import sys + +distfilessrc='./distfiles' + +def hash_file(f): + hsh = hashlib.new('sha1') + with open(f, 'rb') as fle: + hsh.update(fle.read()) + return hsh.hexdigest() + +with os.scandir(path=sys.argv[1]) as it: + for f in it: + if not f.is_file() or f.name.startswith('.'): + continue + srcfile = os.path.join(sys.argv[1], f.name) + h = hash_file(srcfile) + distname = os.path.join(distfilessrc, + f.name + "@" + h).lower() + if os.path.exists(distname): + print("DUP %s" % distname.split('/')[-1]) + os.remove(srcfile) + os.link(distname, srcfile, follow_symlinks=False) + else: + print("NEW %s" % distname.split('/')[-1]) + os.link(srcfile, distname) |