diff options
| -rw-r--r-- | Makefile.gpyutils | 73 | ||||
| -rwxr-xr-x | cgi-bin/get-git-file.sh | 3 | ||||
| -rwxr-xr-x | cgi-bin/trigger-pull.sh | 36 | ||||
| -rwxr-xr-x | check_eclasses_eapis.sh | 17 | ||||
| -rwxr-xr-x | create-dev-keyrings.bash | 10 | ||||
| m--------- | dev-timeline | 0 | ||||
| -rwxr-xr-x | eapi-usage.sh | 86 | ||||
| -rwxr-xr-x | eapi_usage.sh | 25 | ||||
| -rwxr-xr-x | find-binary-files.sh | 6 | ||||
| -rwxr-xr-x | genrdeps-index.py | 3 | ||||
| -rwxr-xr-x | gpy-impl-list | 17 | ||||
| -rw-r--r-- | htdocs/index.html | 60 | ||||
| -rw-r--r-- | htdocs/robots.txt | 2 | ||||
| -rwxr-xr-x | keyrings-export.bash | 69 | ||||
| -rwxr-xr-x | keyrings-recv-keys.gentoo.org.bash | 14 | ||||
| -rwxr-xr-x | keyrings-recv-sks.bash | 1 | ||||
| -rw-r--r-- | keyrings.inc.bash | 36 | ||||
| -rw-r--r-- | kuroneko.exclude | 23 | ||||
| -rwxr-xr-x | leaf-packages.py | 118 | ||||
| m--------- | pkgcheck2html | 0 |
20 files changed, 453 insertions, 146 deletions
diff --git a/Makefile.gpyutils b/Makefile.gpyutils index e7be59e..a2bba56 100644 --- a/Makefile.gpyutils +++ b/Makefile.gpyutils @@ -5,78 +5,47 @@ timestamp = $(PORTDIR)/metadata/timestamp.commit # pass via make! outdir = /dev/null +# Adding new implementations: +# 1. Create rules below & add to upgr_all +# 2. Later, when dot doesn't take too long for it, move to upgr_txt +# 3. Once the impl is stable, move to upgr_base +# +# Please prioritize edges like python-any-r1 consumers to speed up getting +# from 1->2. Especially e.g. git, ninja, meson, doxygen. + # upgrades for stable impls -upgr_base = $(outdir)/36-to-37.txt $(outdir)/37-to-38.txt $(outdir)/38-to-39.txt +upgr_base = $(outdir)/310-to-311.txt # related stabilizations upgr_streq = $(patsubst %.txt,%-stablereq.txt,$(upgr_base)) # all upgrade stuff -upgr_txt = $(upgr_base) $(upgr_streq) $(outdir)/py2.txt $(outdir)/py23.txt +upgr_txt = $(upgr_base) $(upgr_streq) $(outdir)/311-to-312.txt upgr_dot = $(patsubst %.txt,%.dot,$(upgr_txt)) upgr_svg = $(patsubst %.dot,%.svg,$(upgr_dot)) +# add new impls here if not stable yet, to avoid insanely huge generation times upgr_all = $(upgr_txt) $(upgr_dot) $(upgr_svg) all = $(upgr_all) all: $(all) -$(outdir)/cands.txt: $(timestamp) - mkdir -p $(outdir) - gpy-cands > $@.new - mv $@.new $@ - -$(outdir)/depcands.txt: $(timestamp) - mkdir -p $(outdir) - gpy-depcands > $@.new - mv $@.new $@ - -$(outdir)/depcands-all.txt: $(timestamp) +$(outdir)/311-to-312.txt: $(timestamp) mkdir -p $(outdir) - gpy-depcands --all > $@.new + gpy-upgrade-impl -m python3_11 python3_12 > $@.new mv $@.new $@ -$(outdir)/depcheck.txt: $(timestamp) - mkdir -p $(outdir) - gpy-depcheck > $@.new - mv $@.new $@ - -$(outdir)/36-to-37.txt: $(timestamp) - mkdir -p $(outdir) - gpy-upgrade-impl -m python3_6 python3_7 > $@.new - mv $@.new $@ - -$(outdir)/36-to-37-stablereq.txt: $(timestamp) - mkdir -p $(outdir) - gpy-upgrade-impl -m -s python3_6 python3_7 > $@.new - mv $@.new $@ - -$(outdir)/37-to-38.txt: $(timestamp) - mkdir -p $(outdir) - gpy-upgrade-impl -m python3_7 python3_8 > $@.new - mv $@.new $@ - -$(outdir)/37-to-38-stablereq.txt: $(timestamp) - mkdir -p $(outdir) - gpy-upgrade-impl -m -s python3_7 python3_8 > $@.new - mv $@.new $@ - -$(outdir)/38-to-39.txt: $(timestamp) - mkdir -p $(outdir) - gpy-upgrade-impl -m python3_8 python3_9 > $@.new - mv $@.new $@ - -$(outdir)/38-to-39-stablereq.txt: $(timestamp) - mkdir -p $(outdir) - gpy-upgrade-impl -m -s python3_8 python3_9 > $@.new - mv $@.new $@ +#$(outdir)/311-to-312-stablereq.txt: $(timestamp) +# mkdir -p $(outdir) +# gpy-upgrade-impl -m -s python3_11 python3_12 > $@.new +# mv $@.new $@ -$(outdir)/py2.txt: $(timestamp) +$(outdir)/310-to-311.txt: $(timestamp) mkdir -p $(outdir) - gpy-py2 > $@.new + gpy-upgrade-impl -m python3_10 python3_11 > $@.new mv $@.new $@ -$(outdir)/py23.txt: $(timestamp) +$(outdir)/310-to-311-stablereq.txt: $(timestamp) mkdir -p $(outdir) - gpy-py2 -3 > $@.new + gpy-upgrade-impl -m -s python3_10 python3_11 > $@.new mv $@.new $@ %.dot: %.txt diff --git a/cgi-bin/get-git-file.sh b/cgi-bin/get-git-file.sh index 9451d1e..9721f71 100755 --- a/cgi-bin/get-git-file.sh +++ b/cgi-bin/get-git-file.sh @@ -61,7 +61,8 @@ main() { lfile=${file} tree=( $(git ls-tree "${commit}" "${lfile}" 2>/dev/null) ) if [[ ! ${tree[*]} ]]; then - echo "Status: 404 Not Found" + echo "Status: 503 Service Unavailable" + echo "Retry-After: 30" echo echo "404 Not Found (if the report was just published, you may need to wait a minute or two for sync)" exit 0 diff --git a/cgi-bin/trigger-pull.sh b/cgi-bin/trigger-pull.sh new file mode 100755 index 0000000..89aff7a --- /dev/null +++ b/cgi-bin/trigger-pull.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +if [[ ! ${QUERY_STRING} ]]; then + echo "Script must be run through CGI" >&2 + exit 1 +fi + +main() { + local repo=${QUERY_STRING} + if [[ ${repo} == */* ]]; then + echo "DANGER! DANGER! DON'T TALK TO STRANGERS!" >&2 + exit 1 + fi + + local topdir=$(dirname "${0}")/.. + + if ! cd "${topdir}/htdocs/output/${repo}" 2>/dev/null; then + echo "Status: 404 Not Found" + echo + echo "404 Not Found" + exit 0 + fi + + local output=$(git pull -q 2>&1) + if [ $? -eq 0 ]; then + echo "Status: 200 OK" + echo + echo "Done." + else + echo "Status: 500 Failed" + echo + echo "${output}" + fi +} + +main diff --git a/check_eclasses_eapis.sh b/check_eclasses_eapis.sh index 56e2a1b..899b1c3 100755 --- a/check_eclasses_eapis.sh +++ b/check_eclasses_eapis.sh @@ -10,16 +10,17 @@ else fi #[[ $(type pquery 2> /dev/null) ]] || exit 1 +[[ $(type gawk 2> /dev/null) ]] || exit 1 -TMPEAPIS="/tmp/$(basename $0).global.$$.tmp" -TMPECLASS="/tmp/$(basename $0).eclass.$$.tmp" -REPO_PATH=$(portageq get_repo_path / gentoo) +TMPEAPIS="$(mktemp -t $(basename $0).global.tmp.XXXXXX)" +TMPECLASS="$(mktemp -t $(basename $0).eclass.tmp.XXXXXX)" +REPO_PATH=$(portageq get_repo_path ${EROOT:-/} gentoo) pushd "${REPO_PATH}/eclass" > /dev/null ECLASSES=$(echo *.eclass) popd > /dev/null #pquery --attr eapi --attr inherited --raw --all --repo portdir > "${TMPEAPIS}" -find "${REPO_PATH}/metadata/md5-cache" -type f -name '*-[0-9]*' -exec awk -F= ' +find "${REPO_PATH}/metadata/md5-cache" -type f -name '*-[0-9]*' -exec gawk -F= ' BEGINFILE { n = split(FILENAME, f, "/") file = f[n-1] "/" f[n] @@ -38,7 +39,7 @@ find "${REPO_PATH}/metadata/md5-cache" -type f -name '*-[0-9]*' -exec awk -F= ' } ' '{}' \+ > "${TMPEAPIS}" -KNOWN_EAPIS=$(awk ' +KNOWN_EAPIS=$(gawk ' { e = gensub(/eapi="(.*)"/, "\\1", "", $2); eapi[e] = e } END { PROCINFO["sorted_in"]="@ind_num_asc"; for (e in eapi) print e } ' "${TMPEAPIS}") @@ -49,11 +50,11 @@ for x in ${ECLASSES}; do echo "Processing eclass \"${x}\"" rm -rf "${x}" mkdir "${x}" - awk -F'=' '$3 ~ /[ "]'"${x%.eclass}"'[ "]/ {print $1" "$2}' "${TMPEAPIS}" > "${TMPECLASS}" + gawk -F'=' '$3 ~ /[ "]'"${x%.eclass}"'[ "]/ {print $1" "$2}' "${TMPEAPIS}" > "${TMPECLASS}" pushd "${x}" > /dev/null - echo "Overall statistic for eclass \"${x}\":" > "STATS.txt" + echo "Overall statistics for eclass \"${x}\":" > "STATS.txt" for y in ${KNOWN_EAPIS}; do - awk -F ' ' '$3 ~ /"'"${y}"'"/ {print $1}' "${TMPECLASS}" > "${y}.txt" + gawk -F ' ' '$3 ~ /"'"${y}"'"/ {print $1}' "${TMPECLASS}" > "${y}.txt" tmpval=$(wc -l "${y}.txt" |cut -d' ' -f1) echo "EAPI=${y} count: ${tmpval}" >> "STATS.txt" done diff --git a/create-dev-keyrings.bash b/create-dev-keyrings.bash index 65b2b14..3ab1b58 100755 --- a/create-dev-keyrings.bash +++ b/create-dev-keyrings.bash @@ -18,6 +18,10 @@ grab_keys "${SYSTEM_KEYS[@]}" export_keys "${OUTPUT_DIR}"/service-keys.gpg \ "${SYSTEM_KEYS[@]}" +grab_keys "${INFRA_SYSTEM_KEYS[@]}" +export_keys "${OUTPUT_DIR}"/infra-service-keys.gpg \ + "${INFRA_SYSTEM_KEYS[@]}" + grab_keys "${COMMITTING_DEVS[@]}" export_keys "${OUTPUT_DIR}"/committing-devs.gpg \ "${COMMITTING_DEVS[@]}" @@ -27,6 +31,10 @@ export_keys "${OUTPUT_DIR}"/active-devs.gpg \ "${COMMITTING_DEVS[@]}" \ "${NONCOMMITTING_DEVS[@]}" +grab_keys "${INFRA_DEVS[@]}" +export_keys "${OUTPUT_DIR}"/infra-devs.gpg \ + "${INFRA_DEVS[@]}" + # -- not all are on keyservers # -- and are unlikely to turn up now # -- this needs to fetch from some archive instead @@ -39,6 +47,7 @@ export_keys "${OUTPUT_DIR}"/all-devs.gpg \ "${SYSTEM_KEYS[@]}" \ "${COMMITTING_DEVS[@]}" \ "${NONCOMMITTING_DEVS[@]}" \ + "${INFRA_DEVS[@]}" \ "${RETIRED_DEVS[@]}" # Populate keys.gentoo.org with the keys we have, since they might have come from SKS @@ -47,4 +56,5 @@ export KEYSERVER_TIMEOUT=20m push_keys "${SYSTEM_KEYS[@]}" push_keys "${COMMITTING_DEVS[@]}" push_keys "${NONCOMMITTING_DEVS[@]}" +push_keys "${INFRA_DEVS[@]}" push_keys "${RETIRED_DEVS[@]}" diff --git a/dev-timeline b/dev-timeline -Subproject 2335278e82579fb86aab5a9e7e7a31d7f177311 +Subproject 6d256c01a1631c7985a4339ad02bca93390b27f diff --git a/eapi-usage.sh b/eapi-usage.sh new file mode 100755 index 0000000..b7a1112 --- /dev/null +++ b/eapi-usage.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# This script respects EINFO_QUIET=1 etc to silence stdout +# Always puts a summary in $1/README.html. +# Arguments: +# $1: output directory. Defaults to eapi-usage. +# $2: file to place stats in within $1. +. /lib/gentoo/functions.sh + +dir=${1} + +if [[ -n ${1} && -e ${dir} && ! -d ${dir} ]] ; then + eerror "Output directory given (${dir}) is not a directory! Exiting." + exit 1 +elif [[ -z ${dir} ]] ; then + ewarn "No output directory argument given! Defaulting to 'eapi-usage'." + dir=eapi-usage +fi + +stats=${2:-$dir/STATS.txt} + +mkdir -p ${dir} || exit 1 + +REPO_PATH=$(portageq get_repo_path ${EROOT:-/} gentoo || exit 1) +TMPDIR="$(mktemp -d || exit 1)" + +shopt -s nullglob + +einfo "Working in TMPDIR=${TMPDIR}" +pushd "${TMPDIR}" &>/dev/null || exit 1 +mkdir -p eapi-usage || exit 1 +cd eapi-usage || exit 1 + +ebegin "Finding ebuilds" +( + while IFS= read -r ebuild ; do + cpf_eapi="${ebuild#${REPO_PATH}/metadata/md5-cache/}" + cpf="${cpf_eapi%%:*}" + eapi="${cpf_eapi##*:EAPI=}" + echo "${cpf}" >> ${eapi}.txt + done < <(find "${REPO_PATH}/metadata/md5-cache" -mindepth 2 -maxdepth 2 -type f -name '*-[0-9]*' -exec grep '^EAPI=' {} +) +) || { eend $? || exit 1; } +eend ${?} + +ebegin "Sorting EAPI files" +for eapi in *.txt ; do + sort -u ${eapi} > ${eapi}.sorted + mv ${eapi}.sorted ${eapi} +done || { eend $? || exit 1; } +eend $? + +popd &>/dev/null || exit 1 +# No exit here because it's fine if we removed nothing +rm ${dir}/*.txt +mv ${TMPDIR}/eapi-usage/*.txt ${dir}/ || exit 1 + +rm -r "${TMPDIR}" || exit 1 + +# Now generate the numbers/summary (copied in from previous eapi_usage.sh script) +# Someday it would be nice to graph this output, or maybe keep some running history? +# TODO: We used pinspect in the past which is cleaner but it's slower than the +# awk method used below (we also stopped using it because at the time, pkgcore +# lacked newer EAPI support). +#[[ $(type pinspect 2> /dev/null) ]] || exit 1 +# +#pinspect eapi_usage /usr/portage + +find "${REPO_PATH}"/metadata/md5-cache -type f ! -name '*.gz' \ + -exec grep -h '^EAPI=' '{}' + \ + | awk ' + { sub("EAPI=",""); eapi[$1]++ } + END { + PROCINFO["sorted_in"]="@val_num_desc" + for (i in eapi) { + s=""; for (j=1; j<eapi[i]*50./NR+0.5; j++) s=s"#" + printf "EAPI %s: %7d ebuilds (%5.02f%%) %s\n", + i, eapi[i], eapi[i]*100.0/NR, s + } + printf "total: %7d ebuilds\n", NR + }' > ${stats} + +echo >> ${stats} +echo "Date generated: $(date -u '+%Y-%m-%d %H:%M:%S %Z')" >> ${stats} + +echo "<pre>" > ${dir}/README.html +cat ${stats} >> ${dir}/README.html +echo "</pre>" >> ${dir}/README.html diff --git a/eapi_usage.sh b/eapi_usage.sh deleted file mode 100755 index a4a47ac..0000000 --- a/eapi_usage.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# Boring 'script' that just uses pkgcore's pinspect command. Someday it would be -# nice to graph this output, or maybe keep some running history? - -#[[ $(type pinspect 2> /dev/null) ]] || exit 1 -# -#pinspect eapi_usage /usr/portage - -find /usr/portage/metadata/md5-cache -type f ! -name '*.gz' \ - -exec grep -h '^EAPI=' '{}' + \ - | awk ' - { sub("EAPI=",""); eapi[$1]++ } - END { - PROCINFO["sorted_in"]="@val_num_desc" - for (i in eapi) { - s=""; for (j=1; j<eapi[i]*50./NR+0.5; j++) s=s"#" - printf "EAPI %s: %7d ebuilds (%5.02f%%) %s\n", - i, eapi[i], eapi[i]*100.0/NR, s - } - printf "total: %7d ebuilds\n", NR - }' - -echo -echo "Date generated: $(date -u '+%Y-%m-%d %H:%M:%S %Z')" diff --git a/find-binary-files.sh b/find-binary-files.sh index db3ecb8..b053d2f 100755 --- a/find-binary-files.sh +++ b/find-binary-files.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2014-2019 Ulrich Müller +# Copyright 2014-2024 Ulrich Müller # Distributed under the terms of the GNU GPL version 2 or later # Author: Ulrich Müller <ulm@gentoo.org> @@ -33,6 +33,7 @@ while read line; do "image/svg; charset=us-ascii") ;; # SVG image "image/svg+xml; charset=us-ascii") ;; # SVG image "image/x-xpmi; charset=us-ascii") ;; # XPM image + "image/x-xpixmap; charset=us-ascii") ;; # XPM image "message/rfc822; charset=us-ascii") ;; "message/rfc822; charset=utf-8") ;; *) @@ -46,3 +47,6 @@ done < <(find \( -path ./distfiles -o -path ./local -o -path ./metadata \ -exec file -ih '{}' + | sort) [[ ${count} -gt 0 ]] || echo "No binary files found. :-)" + +# Output the file version for debugging of false positives/negatives +echo; file --version | head -n1 diff --git a/genrdeps-index.py b/genrdeps-index.py index 1dc958b..9b1cd85 100755 --- a/genrdeps-index.py +++ b/genrdeps-index.py @@ -25,6 +25,7 @@ DepTuple = collections.namedtuple('DepTuple', ('cpv', 'blocks', 'use')) GROUPS = ( ('bdepend', 'bindex'), ('depend', 'dindex'), + ('idepend', 'iindex'), ('pdepend', 'pindex'), ('rdepend', 'rindex'), ) @@ -63,7 +64,7 @@ def main(): args = argp.parse_args() c = pkgcore.config.load_config() - repo = c.repo['gentoo'] + repo = c.objects.repo['gentoo'] rindex = {} for g, gi in GROUPS: diff --git a/gpy-impl-list b/gpy-impl-list new file mode 100755 index 0000000..be70554 --- /dev/null +++ b/gpy-impl-list @@ -0,0 +1,17 @@ +#!/bin/bash + +OUTDIR=$1 + +if [[ -z ${OUTDIR} ]]; then + echo "Usage: $0 <out-dir>" >&2 + exit 1 +fi + +IRC='ircs://irc.libera.chat/#gentoo-python' + +irk "${IRC}" - <<<$'\0' +gpy-list-pkg-impls > "${OUTDIR}"/python-pkg-impls.txt.new || exit 1 +while read -r x; do + irk "${IRC}" "${x}" +done < <(git diff --no-index -s --word-diff -U0 "${OUTDIR}"/python-pkg-impls.txt{,.new} | tail -n +5 | grep -v '^@@') +mv "${OUTDIR}"/python-pkg-impls.txt{.new,} diff --git a/htdocs/index.html b/htdocs/index.html index 4cfd715..f40051e 100644 --- a/htdocs/index.html +++ b/htdocs/index.html @@ -91,7 +91,7 @@ <div class="list-group mb-3"> <a href="/output/wot-graph.svg" class="list-group-item list-group-item-action">Gentoo Dev Web of Trust (WoT)</a> <a href="/output/wot-stats.html" class="list-group-item list-group-item-action">WoT Node Stats</a> - <a href="/output/maintainer-needed.html" class="list-group-item list-group-item-action">"Maintainer Needed" Packages</a> + <a href="/output/maintainer-needed.html" class="list-group-item list-group-item-action">"Maintainer Needed" (orphaned) Packages</a> <a href="/output/dev-timeline.html" class="list-group-item list-group-item-action">Developer commit timeline</a> <a href="/output/active-devs.html" class="list-group-item list-group-item-action">Active developer commit timeline</a> <a href="/output/repos/" class="list-group-item list-group-item-action">Repository status reports</a> @@ -100,59 +100,45 @@ <h3>Python reports</h3> <div class="list-group mb-3"> <div class="list-group-item list-group-item-action p-0"> - <a href="/output/gpyutils/36-to-37.txt" class="list-group-item-base-link">Py3.6 → 3.7 testing</a> - <a href="/output/gpyutils/36-to-37.svg" class="list-group-item-additional-link p-1"> + <a href="/output/gpyutils/311-to-312.txt" class="list-group-item-base-link">Py3.11 → 3.12 testing</a> + <a href="/output/gpyutils/311-to-312.svg" class="list-group-item-additional-link p-1"> <span class="badge badge-primary">svg</span> </a> </div> <div class="list-group-item list-group-item-action p-0"> - <a href="/output/gpyutils/36-to-37-stablereq.txt" class="list-group-item-base-link">Py3.6 → 3.7 stabilizations</a> - <a href="/output/gpyutils/36-to-37-stablereq.svg" class="list-group-item-additional-link p-1"> + <a href="/output/gpyutils/310-to-311.txt" class="list-group-item-base-link">Py3.10 → 3.11 testing</a> + <a href="/output/gpyutils/310-to-311.svg" class="list-group-item-additional-link p-1"> <span class="badge badge-primary">svg</span> </a> - </div> - <div class="list-group-item list-group-item-action p-0"> - <a href="/output/gpyutils/37-to-38.txt" class="list-group-item-base-link">Py3.7 → 3.8 testing</a> - <a href="/output/gpyutils/37-to-38.svg" class="list-group-item-additional-link p-1"> - <span class="badge badge-primary">svg</span> - </a> - </div> + </div> <div class="list-group-item list-group-item-action p-0"> - <a href="/output/gpyutils/37-to-38-stablereq.txt" class="list-group-item-base-link">Py3.7 → 3.8 stabilizations</a> - <a href="/output/gpyutils/37-to-38-stablereq.svg" class="list-group-item-additional-link p-1"> - <span class="badge badge-primary">svg</span> - </a> - </div> - <div class="list-group-item list-group-item-action p-0"> - <a href="/output/gpyutils/38-to-39.txt" class="list-group-item-base-link">Py3.8 → 3.9 testing</a> - <a href="/output/gpyutils/38-to-39.svg" class="list-group-item-additional-link p-1"> - <span class="badge badge-primary">svg</span> - </a> - </div> - <div class="list-group-item list-group-item-action p-0"> - <a href="/output/gpyutils/py2.txt" class="list-group-item-base-link">Py2-only packages</a> - <a href="/output/gpyutils/py2.svg" class="list-group-item-additional-link p-1"> - <span class="badge badge-primary">svg</span> - </a> - </div> - <div class="list-group-item list-group-item-action p-0"> - <a href="/output/gpyutils/py23.txt" class="list-group-item-base-link">Py2+3 packages</a> - <a href="/output/gpyutils/py23.svg" class="list-group-item-additional-link p-1"> + <a href="/output/gpyutils/310-to-311-stablereq.txt" class="list-group-item-base-link">Py3.10 → 3.11 stabilizations</a> + <a href="/output/gpyutils/310-to-311-stablereq.svg" class="list-group-item-additional-link p-1"> <span class="badge badge-primary">svg</span> </a> </div> </div> + <h3>Generated artifacts</h3> + <div class="list-group mb-3"> + <a href="/output/service-keys.gpg" class="list-group-item list-group-item-action">GPG keyring: service keys</a> + <a href="/output/active-devs.gpg" class="list-group-item list-group-item-action">GPG keyring: all active devs</a> + <a href="/output/committing-devs.gpg" class="list-group-item list-group-item-action">GPG keyring: active gentoo.git devs</a> + <a href="/output/retired-devs.gpg" class="list-group-item list-group-item-action">GPG keyring: retired devs</a> + <a href="/output/infra-devs.gpg" class="list-group-item list-group-item-action">GPG keyring: infra devs</a> + <a href="/output/all-devs.gpg" class="list-group-item list-group-item-action">GPG keyring: all devs</a> + </div> </div> <div class="col-lg-4"> <h3>Tree-wide Scanning</h3> <div class="list-group mb-3"> - <a href="/output/eapi_usage.txt" class="list-group-item list-group-item-action">EAPI Usage</a> + <a href="/output/eapi-usage/" class="list-group-item list-group-item-action">EAPI Usage</a> <a href="/output/eapi-per-eclass" class="list-group-item list-group-item-action">EAPI Usage per eclass</a> <a href="/output/eapi-per-eclass/matrix.txt" class="list-group-item list-group-item-action">Eclass-EAPI usage matrix</a> <a href="/output/invalid-mask.txt" class="list-group-item list-group-item-action">Invalid package.mask entries</a> <a href="/output/filesdir-usage.txt" class="list-group-item list-group-item-action">Top 100 largest FILESDIRs</a> - <a href="/output/find-binary-files.txt" class="list-group-item list-group-item-action">Binary files in gentoo-x86</a> + <a href="/output/find-binary-files.txt" class="list-group-item list-group-item-action">Binary files in ::gentoo</a> <a href="/output/eclass-usage/" class="list-group-item list-group-item-action">Listing of packages that inherit an eclass</a> + <a href="/output/leaf-packages.txt" class="list-group-item list-group-item-action">Leaf Packages</a> <div class="list-group-item list-group-item-action p-0"> <a href="/output/gentoo-ci/output.html" class="list-group-item-base-link">Gentoo CI – QA check results</a> <a href="/output/gentoo-ci/output.verbose.html" class="list-group-item-additional-link p-1"> @@ -165,6 +151,7 @@ <div class="list-group mb-3"> <a href="/output/genrdeps/rindex/" class="list-group-item list-group-item-action">Reverse RDEPEND</a> <a href="/output/genrdeps/dindex/" class="list-group-item list-group-item-action">Reverse DEPEND</a> + <a href="/output/genrdeps/iindex/" class="list-group-item list-group-item-action">Reverse IDEPEND</a> <a href="/output/genrdeps/pindex/" class="list-group-item list-group-item-action">Reverse PDEPEND</a> <a href="/output/genrdeps/bindex/" class="list-group-item list-group-item-action">Reverse BDEPEND</a> </div> @@ -185,6 +172,9 @@ <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2018-01-01&v2=2017-01-01" class="list-group-item list-group-item-action">2017</a> <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2019-01-01&v2=2018-01-01" class="list-group-item list-group-item-action">2018</a> <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2020-01-01&v2=2019-01-01" class="list-group-item list-group-item-action">2019</a> + <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2021-01-01&v2=2020-01-01" class="list-group-item list-group-item-action">2020</a> + <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2022-01-01&v2=2021-01-01" class="list-group-item list-group-item-action">2021</a> + <a href="https://bugs.gentoo.org/buglist.cgi?f1=delta_ts&list_id=1554870&o1=lessthaneq&resolution=---&o2=greaterthaneq&query_format=advanced&f2=delta_ts&v1=2023-01-01&v2=2022-01-01" class="list-group-item list-group-item-action">2022</a> </div> </div> </div> @@ -212,7 +202,7 @@ </ul> </div> <div class="col-xs-10 col-sm-9 col-md-10"> - <strong>© 2001–2020 Gentoo Authors</strong><br> + <strong>© 2001–2023 Gentoo Authors</strong><br> <small> Gentoo is a trademark of the Gentoo Foundation, Inc. The contents of this document, unless otherwise expressly stated, are licensed under the diff --git a/htdocs/robots.txt b/htdocs/robots.txt new file mode 100644 index 0000000..1f53798 --- /dev/null +++ b/htdocs/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/keyrings-export.bash b/keyrings-export.bash index 5b6dc17..56e4e51 100755 --- a/keyrings-export.bash +++ b/keyrings-export.bash @@ -6,6 +6,8 @@ # - requires keeping state to detect changes in keys, there is no usable mtime data in a key itself OUTPUT_DIR=${1:-.} +# Ensure output is absolute +OUTPUT_DIR=$(readlink -f "${OUTPUT_DIR}") BASEDIR="$(dirname "$0")" # shellcheck source=./keyrings.inc.bash source "${BASEDIR}"/keyrings.inc.bash @@ -16,25 +18,72 @@ export -a COMMITTING_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${COMMIT_ export -a NONCOMMITTING_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${NONCOMMIT_RULE}") ) export -a RETIRED_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${RETIRED_RULE}") ) export -a SYSTEM_KEYS=( $(grab_ldap_fingerprints -b "${SYSTEM_BASE}" "${NONCOMMIT_RULE}") ) +export -a INFRA_SYSTEM_KEYS=( $(grab_ldap_fingerprints -b "${SYSTEM_BASE}" "${INFRA_SYSTEM_RULE}") ) +export -a KEYRINGS=( ) -export_keys "${OUTPUT_DIR}"/service-keys.gpg \ - "${SYSTEM_KEYS[@]}" +export_keys "${OUTPUT_DIR}"/keys/service-keys.gpg \ + "${SYSTEM_KEYS[@]}" \ +&& KEYRINGS+=( service-keys ) + +export_keys "${OUTPUT_DIR}"/keys/infra-service-keys.gpg \ + "${INFRA_SYSTEM_KEYS[@]}" \ +&& KEYRINGS+=( infra-service-keys ) -export_keys "${OUTPUT_DIR}"/committing-devs.gpg \ - "${COMMITTING_DEVS[@]}" +export_keys "${OUTPUT_DIR}"/keys/committing-devs.gpg \ + "${COMMITTING_DEVS[@]}" \ +&& KEYRINGS+=( committing-devs ) -export_keys "${OUTPUT_DIR}"/active-devs.gpg \ +export_keys "${OUTPUT_DIR}"/keys/active-devs.gpg \ "${COMMITTING_DEVS[@]}" \ - "${NONCOMMITTING_DEVS[@]}" + "${NONCOMMITTING_DEVS[@]}" \ +&& KEYRINGS+=( active-devs ) -export_keys "${OUTPUT_DIR}"/retired-devs.gpg \ - "${RETIRED_DEVS[@]}" +export_keys "${OUTPUT_DIR}"/keys/infra-devs.gpg \ + "${INFRA_DEVS[@]}" \ +&& KEYRINGS+=( infra-devs ) + +export_keys "${OUTPUT_DIR}"/keys/retired-devs.gpg \ + "${RETIRED_DEVS[@]}" \ +&& KEYRINGS+=( retired-devs ) # Everybody together now -export_keys "${OUTPUT_DIR}"/all-devs.gpg \ +export_keys "${OUTPUT_DIR}"/keys/all-devs.gpg \ "${SYSTEM_KEYS[@]}" \ + "${INFRA_SYSTEM_KEYS[@]}" \ "${COMMITTING_DEVS[@]}" \ "${NONCOMMITTING_DEVS[@]}" \ - "${RETIRED_DEVS[@]}" + "${INFRA_DEVS[@]}" \ + "${RETIRED_DEVS[@]}" \ +&& KEYRINGS+=( all-devs ) + +for key in "${KEYRINGS[@]}" ; do + if [[ ! -L "${OUTPUT_DIR}"/${key}.gpg ]] ; then + # Compatibility symlink + ln -sf "${OUTPUT_DIR}"/keys/${key}.gpg "${OUTPUT_DIR}"/${key}.gpg + fi + + if [[ $(date -u +%A) == Monday ]] ; then + # We don't want to run on Mondays to avoid last/next week confusion + break + fi + + timestamp=$(date -u +%Y%m%d-%A -d "last monday") + + if [[ ${timestamp} != *-Monday ]] ; then + break + fi + + timestamp=${timestamp/-Monday/} + + # Don't clobber existing timestamped keys for this period (weekly) + # if we're running several times a day. + if [[ -f "${OUTPUT_DIR}"/keys/${key}-${timestamp}.gpg ]] ; then + continue + fi + + mkdir -p "${OUTPUT_DIR}"/keys + + cp "${OUTPUT_DIR}"/${key}.gpg "${OUTPUT_DIR}"/keys/${key}-${timestamp}.gpg +done clean_tmp diff --git a/keyrings-recv-keys.gentoo.org.bash b/keyrings-recv-keys.gentoo.org.bash index 6f3ac0e..7cc1b62 100755 --- a/keyrings-recv-keys.gentoo.org.bash +++ b/keyrings-recv-keys.gentoo.org.bash @@ -1,16 +1,17 @@ #!/bin/bash # Import key updates from Keyservers: keys.gentoo.org -# -# TODO: -# - Turn off export in this script BASEDIR="$(dirname "$0")" +DEBUG=${DEBUG:=0} # shellcheck source=./keyrings.inc.bash source "${BASEDIR}"/keyrings.inc.bash set -e -export_ldap_data_to_env + +# export_ldap_data_to_env +# TODO: for unclear reason this does not populate correctly inside a function export -a COMMITTING_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${COMMIT_RULE}") ) +export -a INFRA_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${INFRA_RULE}") ) export -a NONCOMMITTING_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${NONCOMMIT_RULE}") ) export -a RETIRED_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${RETIRED_RULE}") ) export -a SYSTEM_KEYS=( $(grab_ldap_fingerprints -b "${SYSTEM_BASE}" "${NONCOMMIT_RULE}") ) @@ -18,9 +19,14 @@ export -a SYSTEM_KEYS=( $(grab_ldap_fingerprints -b "${SYSTEM_BASE}" "${NONCOMMI export KEYSERVERS=( "${KS_GENTOO}" ) export KEYSERVER_TIMEOUT=5m +[[ $DEBUG -ne 0 ]] && echo SYSTEM_KEYS grab_keys "${SYSTEM_KEYS[@]}" +[[ $DEBUG -ne 0 ]] && echo COMITTING_DEVS grab_keys "${COMMITTING_DEVS[@]}" +[[ $DEBUG -ne 0 ]] && echo NONCOMITTING_DEVS grab_keys "${NONCOMMITTING_DEVS[@]}" +[[ $DEBUG -ne 0 ]] && echo INFRA_DEVS +grab_keys "${INFRA_DEVS[@]}" # -- not all are on keyservers # -- and are unlikely to turn up now # -- this needs to fetch from some archive instead diff --git a/keyrings-recv-sks.bash b/keyrings-recv-sks.bash index 5ab91c3..81ba0a2 100755 --- a/keyrings-recv-sks.bash +++ b/keyrings-recv-sks.bash @@ -21,6 +21,7 @@ export KEYSERVER_TIMEOUT=20m grab_keys "${SYSTEM_KEYS[@]}" grab_keys "${COMMITTING_DEVS[@]}" grab_keys "${NONCOMMITTING_DEVS[@]}" +grab_keys "${INFRA_DEVS[@]}" # -- not all are on keyservers # -- and are unlikely to turn up now # -- this needs to fetch from some archive instead diff --git a/keyrings.inc.bash b/keyrings.inc.bash index d19f0ad..d2668e0 100644 --- a/keyrings.inc.bash +++ b/keyrings.inc.bash @@ -6,14 +6,21 @@ SYSTEM_BASE='ou=system,dc=gentoo,dc=org' COMMIT_RULE='(&(gentooAccess=git.gentoo.org/repo/gentoo.git)(gentooStatus=active))' NONCOMMIT_RULE='(&(!(gentooAccess=git.gentoo.org/repo/gentoo.git))(gentooStatus=active))' RETIRED_RULE='(!(gentooStatus=active))' +INFRA_RULE='(&(gentooAccess=infra.group)(gentooStatus=active))' +INFRA_SYSTEM_RULE='(&(gentooAccess=infra-system.group)(gentooStatus=active))' export KS_GENTOO=hkps://keys.gentoo.org/ +# Use local keyserver for speedup +# KS_GENTOO_LOCAL=${HOSTNAME%.gentoo.org}.keys.gentoo.org +# dig $KS_GENTOO_LOCAL +short |grep -sq . && export KS_GENTOO=hkps://${KS_GENTOO_LOCAL} + #export KS_SKS=hkps://hkps.pool.sks-keyservers.net/ # Disabled pending security announcement export KS_OPENPGP=hkps://keys.openpgp.org/ # runs Hagrid export KEYSERVERS=( ) # empty by default export COMMITTING_DEVS=( ) export NONCOMMITTING_DEVS=( ) export RETIRED_DEVS=( ) +export INFRA_DEVS=( ) export SYSTEM_KEYS=( ) # grab_ldap_fingerprints <ldap-rule> @@ -29,7 +36,7 @@ grab_keys() { local missing=() local remaining=( "${@}" ) - KEYSERVER_TIMEOUT=${KEYSERVER_TIMEOUT:=1m} + KEYSERVER_TIMEOUT=${KEYSERVER_TIMEOUT:=2m} # quickly handle empty keyservers set [ "${#KEYSERVERS[@]}" -eq 0 ] && return while :; do @@ -73,6 +80,8 @@ push_keys() { export GPG_TMPDIR='' clean_tmp() { + # Ensure any agent is closed down + gpgconf --kill all [ -n "$GPG_TMPDIR" ] && [ -d "$GPG_TMPDIR" ] && rm -rf "$GPG_TMPDIR" } setup_tmp() { @@ -87,7 +96,8 @@ export_keys() { DST="$1" shift setup_tmp - TMP="${GPG_TMPDIR}"/$(basename "${DST}") + BASENAME=$(basename "${DST}") + TMP="${GPG_TMPDIR}/${BASENAME}" # Must not exist, otherwise GPG will give error [[ -f "${TMP}" ]] && rm -f "${TMP}" # 'gpg --export' returns zero if there was no error with the command itself @@ -107,15 +117,21 @@ export_keys() { echo "Unable to export keys to $DST: GPG failed to list packets" exit 1 fi + + # Ensure we have a checksum to verify the file. + rhash --bsd --sha256 --sha512 --blake2b "${TMP}" |sed "s,${TMP},${BASENAME},g" >"${TMP}.DIGESTS" + # Check if the textual format has changed at all, and emit the new version # if there are ANY changes at all. - if ! cmp -s "${DST}.packets.txt" "${TMP}.packets.txt"; then - chmod a+r "${TMP}" - mv -f "${TMP}" "${DST}" - mv -f "${TMP}.packets.txt" "${DST}.packets.txt" - fi - # Cleanup anyway - rm -f "${TMP}.packets.txt" "${TMP}" + cmp -s "${DST}.packets.txt" "${TMP}.packets.txt" + cmp_rc=$? + chmod a+r "${TMP}" "${TMP}.packets.txt" "${TMP}.DIGESTS" + for suffix in '' '.packets.txt' '.DIGESTS'; do + # If these file do not exist, or the cmp was different, move them. + [ ! -f "${DST}${suffix}" -o $cmp_rc -ne 0 ] && mv -f "${TMP}${suffix}" "${DST}${suffix}" + # Cleanup anyway + rm -f "${TMP}${suffix}" + done } # populate common variables @@ -124,5 +140,7 @@ export_ldap_data_to_env() { export -a COMMITTING_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${COMMIT_RULE}") ) export -a NONCOMMITTING_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${NONCOMMIT_RULE}") ) export -a RETIRED_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${RETIRED_RULE}") ) + export -a INFRA_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${INFRA_RULE}") ) export -a SYSTEM_KEYS=( $(grab_ldap_fingerprints -b "${SYSTEM_BASE}" "${NONCOMMIT_RULE}") ) + export -a INFRA_SYSTEM_KEYS=( $(grab_ldap_fingerprints -b "${SYSTEM_BASE}" "${INFRA_SYSTEM_RULE}") ) } diff --git a/kuroneko.exclude b/kuroneko.exclude new file mode 100644 index 0000000..3cc605f --- /dev/null +++ b/kuroneko.exclude @@ -0,0 +1,23 @@ +# upstream restarted versioning +404095 +421031 +429522 +433389 +438750 +442478 +452432 +461714 +466822 +499464 +501472 +508270 +524560 +537940 +559532 +565842 +567850 +572716 +578300 +581028 +609562 +619458 diff --git a/leaf-packages.py b/leaf-packages.py new file mode 100755 index 0000000..1454eb8 --- /dev/null +++ b/leaf-packages.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +# From Leo3418's GSoC 2021 work +# https://leo3418.github.io/2021/07/18/find-leaf-packages.html + +import concurrent.futures +import os +import re +import subprocess +import sys + +METHOD="pkgcore" +VERBOSE=int(os.getenv('VERBOSE') or os.getenv('V') or 0) # V=1..N to use + +def main() -> None: + if len(sys.argv) > 1: + repo = sys.argv[1] + else: + repo = 'gentoo' + zero_in_degree = create_ebuild_dict(repo) + with concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count()) \ + as executor: + for ebuild in zero_in_degree: + # Let the executor run function call + # update_for_deps_of(ebuild, zero_in_degree) + if METHOD == "pkgcore": + executor.submit(update_for, ebuild, zero_in_degree, repo) + else: + executor.submit(update_for_deps_of, ebuild, zero_in_degree) + + # Print leaf ebuilds to standard output + for ebuild in zero_in_degree: + if zero_in_degree[ebuild]: + print(ebuild) + + +def update_for(ebuild: str, zero_in_degree: dict, repo: str) -> None: + """ + Update the boolean value for the specified ebuild in the given dictionary. + Reverse dependencies of the ebuild will be searched in the specified + repository only. + """ + # This spams infra cronjobs, only enable if there is a problem + if VERBOSE >= 1: + print(f"Processing {ebuild} ...", file=sys.stderr) + proc = subprocess.run(f'pquery --first --restrict-revdep ={ebuild} ' + f'--repo {repo} --raw --unfiltered', + capture_output=True, text=True, shell=True) + zero_in_degree[ebuild] = len(proc.stdout) == 0 + +def create_ebuild_dict(repo: str) -> dict: + """ + Create a dictionary with all ebuilds in the specified repository as keys + that maps each key to a boolean value indicating whether it is a leaf + ebuild with zero in-degree. + """ + zero_in_degree = {} + proc = subprocess.run(f'pquery --repo {repo} --raw --unfiltered', + capture_output=True, text=True, + shell=True, check=True) + ebuilds = proc.stdout.splitlines() + for ebuild in ebuilds: + zero_in_degree[ebuild] = True + return zero_in_degree + + +def update_for_deps_of(ebuild: str, zero_in_degree: dict) -> None: + """ + For ebuilds that can be pulled as the specified ebuild's dependencies, + update the boolean value for them in the given dictionary accordingly. + """ + + def get_dep_atoms() -> list: + """ + Return a list of all dependency specification atoms. + """ + dep_atoms = [] + equery_dep_atom_pattern = re.compile(r'\(.+/.+\)') + proc = subprocess.run(f'equery -CN depgraph -MUl {ebuild}', + capture_output=True, text=True, shell=True) + out_lines = proc.stdout.splitlines() + for line in out_lines: + dep_atom_match = equery_dep_atom_pattern.findall(line) + dep_atom = [dep.strip('()') for dep in dep_atom_match] + dep_atoms.extend(dep_atom) + return dep_atoms + + def find_matching_ebuilds(atom: str) -> list: + """ + Return a list of ebuilds that satisfy an atom. + """ + proc = subprocess.run(f"equery list -op -F '$cpv' '{atom}'", + capture_output=True, text=True, shell=True) + return proc.stdout.splitlines() + + if VERBOSE >= 2: + print(f"Processing {ebuild} ...", file=sys.stderr) + + # Get dependency specifications in the ebuild; + # equivalent to dep_graph[ebuild] in the examples above + dep_atoms = get_dep_atoms() + + # Convert list of atoms to list of ebuilds that satisfy them + dep_ebuilds = [] + for dep_atom in dep_atoms: + dep_ebuilds.extend(find_matching_ebuilds(dep_atom)) + + # Register dependency ebuilds as non-leaves + for dep_ebuild in dep_ebuilds: + # An ebuild in an overlay might depend on ebuilds from ::gentoo and/or + # other repositories, but we only care about ebuilds in the dictionary + # passed to this function + if dep_ebuild in zero_in_degree: + zero_in_degree[dep_ebuild] = False + + +if __name__ == '__main__': + main() + diff --git a/pkgcheck2html b/pkgcheck2html -Subproject 660808b458ccce2eabd320f2201803c6d10b6a9 +Subproject bb9807de3be23ef879e492c0142e770d54ff450 |