aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--.github/workflows/ci.yml2
-rw-r--r--.gitignorerevs2
-rw-r--r--DEVELOPING19
-rw-r--r--MANIFEST.in3
-rw-r--r--NEWS15
-rw-r--r--RELEASE-NOTES9
-rwxr-xr-xbin/ebuild-ipc.py6
-rwxr-xr-xbin/ebuild.sh39
-rwxr-xr-xbin/estrip99
-rwxr-xr-xbin/gpkg-helper.py71
-rw-r--r--bin/isolated-functions.sh39
-rwxr-xr-xbin/misc-functions.sh67
-rw-r--r--bin/phase-functions.sh4
-rw-r--r--bin/phase-helpers.sh14
-rwxr-xr-xbin/quickpkg109
-rw-r--r--cnf/make.conf.example36
-rw-r--r--cnf/make.conf.example.riscv.diff61
-rw-r--r--cnf/make.globals23
-rw-r--r--lib/_emerge/AbstractEbuildProcess.py4
-rw-r--r--lib/_emerge/Binpkg.py28
-rw-r--r--lib/_emerge/BinpkgExtractorAsync.py9
-rw-r--r--lib/_emerge/BinpkgFetcher.py31
-rw-r--r--lib/_emerge/EbuildBinpkg.py16
-rw-r--r--lib/_emerge/EbuildBuild.py19
-rw-r--r--lib/_emerge/EbuildIpcDaemon.py2
-rw-r--r--lib/_emerge/EbuildPhase.py33
-rw-r--r--lib/_emerge/Package.py1
-rw-r--r--lib/_emerge/actions.py77
-rw-r--r--lib/_emerge/depgraph.py57
-rw-r--r--lib/_emerge/main.py11
-rw-r--r--lib/portage/__init__.py1
-rw-r--r--lib/portage/binpkg.py56
-rw-r--r--lib/portage/const.py13
-rw-r--r--lib/portage/dbapi/bintree.py446
-rw-r--r--lib/portage/dbapi/vartree.py77
-rw-r--r--lib/portage/dep/dep_check.py6
-rw-r--r--lib/portage/dep/soname/multilib_category.py26
-rw-r--r--lib/portage/exception.py20
-rw-r--r--lib/portage/gpg.py106
-rw-r--r--lib/portage/gpkg.py2015
-rw-r--r--lib/portage/output.py6
-rw-r--r--lib/portage/package/ebuild/_config/special_env_vars.py1
-rw-r--r--lib/portage/package/ebuild/config.py10
-rw-r--r--lib/portage/package/ebuild/doebuild.py28
-rw-r--r--lib/portage/package/ebuild/fetch.py7
-rw-r--r--lib/portage/package/ebuild/prepare_build_dirs.py9
-rw-r--r--lib/portage/process.py2
-rw-r--r--lib/portage/tests/.gnupg/openpgp-revocs.d/06B3A311BD775C280D22A9305D90EA06352177F6.rev37
-rw-r--r--lib/portage/tests/.gnupg/openpgp-revocs.d/8DEDA2CDED49C8809287B89D8812797DDF1DD192.rev37
-rw-r--r--lib/portage/tests/.gnupg/private-keys-v1.d/273B030399E7BEA66A9AD42216DE7CA17BA5D42E.keybin0 -> 2055 bytes
-rw-r--r--lib/portage/tests/.gnupg/private-keys-v1.d/C99796FB85B0C3DF03314A11B5850C51167D6282.keybin0 -> 2055 bytes
-rw-r--r--lib/portage/tests/.gnupg/pubring.kbxbin0 -> 2774 bytes
-rw-r--r--lib/portage/tests/.gnupg/trustdb.gpgbin0 -> 1360 bytes
-rw-r--r--lib/portage/tests/__init__.py37
-rw-r--r--lib/portage/tests/ebuild/test_doebuild_spawn.py1
-rw-r--r--lib/portage/tests/ebuild/test_ipc_daemon.py6
-rw-r--r--lib/portage/tests/emerge/test_simple.py47
-rw-r--r--lib/portage/tests/gpkg/__init__.py2
-rw-r--r--lib/portage/tests/gpkg/__test__.py0
-rw-r--r--lib/portage/tests/gpkg/test_gpkg_checksum.py396
-rw-r--r--lib/portage/tests/gpkg/test_gpkg_gpg.py398
-rw-r--r--lib/portage/tests/gpkg/test_gpkg_metadata_update.py59
-rw-r--r--lib/portage/tests/gpkg/test_gpkg_metadata_url.py173
-rw-r--r--lib/portage/tests/gpkg/test_gpkg_path.py390
-rw-r--r--lib/portage/tests/gpkg/test_gpkg_size.py58
-rw-r--r--lib/portage/tests/gpkg/test_gpkg_stream.py112
-rw-r--r--lib/portage/tests/resolver/ResolverPlayground.py55
-rw-r--r--lib/portage/tests/resolver/binpkg_multi_instance/test_build_id_profile_format.py50
-rw-r--r--lib/portage/tests/resolver/binpkg_multi_instance/test_rebuilt_binaries.py44
-rw-r--r--lib/portage/tests/resolver/soname/test_autounmask.py38
-rw-r--r--lib/portage/tests/resolver/soname/test_downgrade.py84
-rw-r--r--lib/portage/tests/resolver/soname/test_or_choices.py39
-rw-r--r--lib/portage/tests/resolver/soname/test_reinstall.py40
-rw-r--r--lib/portage/tests/resolver/soname/test_skip_update.py39
-rw-r--r--lib/portage/tests/resolver/soname/test_slot_conflict_reinstall.py135
-rw-r--r--lib/portage/tests/resolver/soname/test_slot_conflict_update.py38
-rw-r--r--lib/portage/tests/resolver/soname/test_soname_provided.py45
-rw-r--r--lib/portage/tests/resolver/soname/test_unsatisfiable.py40
-rw-r--r--lib/portage/tests/resolver/soname/test_unsatisfied.py40
-rw-r--r--lib/portage/tests/resolver/test_autounmask_binpkg_use.py38
-rw-r--r--lib/portage/tests/resolver/test_bdeps.py44
-rw-r--r--lib/portage/tests/resolver/test_binary_pkg_ebuild_visibility.py35
-rw-r--r--lib/portage/tests/resolver/test_changed_deps.py41
-rw-r--r--lib/portage/tests/resolver/test_complete_if_new_subslot_without_revbump.py40
-rw-r--r--lib/portage/tests/resolver/test_disjunctive_depend_order.py34
-rw-r--r--lib/portage/tests/resolver/test_multirepo.py62
-rw-r--r--lib/portage/tests/resolver/test_regular_slot_change_without_revbump.py41
-rw-r--r--lib/portage/tests/resolver/test_simple.py34
-rw-r--r--lib/portage/tests/resolver/test_slot_abi.py113
-rw-r--r--lib/portage/tests/resolver/test_slot_abi_downgrade.py77
-rw-r--r--lib/portage/tests/resolver/test_slot_change_without_revbump.py40
-rw-r--r--lib/portage/tests/resolver/test_slot_operator_autounmask.py40
-rw-r--r--lib/portage/tests/resolver/test_slot_operator_bdeps.py74
-rw-r--r--lib/portage/tests/resolver/test_slot_operator_rebuild.py40
-rw-r--r--lib/portage/tests/resolver/test_slot_operator_unsolved.py41
-rw-r--r--lib/portage/tests/resolver/test_unecessary_slot_upgrade.py62
-rw-r--r--lib/portage/tests/resolver/test_useflags.py37
-rwxr-xr-xlib/portage/tests/runTests.py12
-rw-r--r--lib/portage/tests/update/test_move_ent.py129
-rw-r--r--lib/portage/tests/update/test_move_slot_ent.py139
-rw-r--r--lib/portage/tests/update/test_update_dbentry.py182
-rw-r--r--lib/portage/util/_dyn_libs/dyn_libs.py28
-rw-r--r--lib/portage/util/_urlopen.py4
-rw-r--r--lib/portage/util/_xattr.py1
-rw-r--r--lib/portage/util/elf/constants.py9
-rw-r--r--lib/portage/util/futures/unix_events.py1
-rw-r--r--lib/portage/versions.py16
-rw-r--r--man/color.map.56
-rw-r--r--man/emerge.17
-rw-r--r--man/make.conf.587
-rw-r--r--repoman/lib/repoman/tests/.gnupg/openpgp-revocs.d/06B3A311BD775C280D22A9305D90EA06352177F6.rev37
-rw-r--r--repoman/lib/repoman/tests/.gnupg/openpgp-revocs.d/8DEDA2CDED49C8809287B89D8812797DDF1DD192.rev37
-rw-r--r--repoman/lib/repoman/tests/.gnupg/private-keys-v1.d/273B030399E7BEA66A9AD42216DE7CA17BA5D42E.keybin0 -> 2055 bytes
-rw-r--r--repoman/lib/repoman/tests/.gnupg/private-keys-v1.d/C99796FB85B0C3DF03314A11B5850C51167D6282.keybin0 -> 2055 bytes
-rw-r--r--repoman/lib/repoman/tests/.gnupg/pubring.kbxbin0 -> 2774 bytes
-rw-r--r--repoman/lib/repoman/tests/.gnupg/trustdb.gpgbin0 -> 1360 bytes
-rw-r--r--repoman/lib/repoman/tests/runTests.py10
-rwxr-xr-xrepoman/setup.py12
-rwxr-xr-xsetup.py24
-rw-r--r--src/portage_util_file_copy_reflink_linux.c7
120 files changed, 6758 insertions, 1058 deletions
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a8466e0f1..581b2d04d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -17,7 +17,7 @@ jobs:
- '3.7'
- '3.8'
- '3.9'
- - '3.10.0-alpha - 3.10.0'
+ - '3.10'
- 'pypy-3.7'
steps:
diff --git a/.gitignorerevs b/.gitignorerevs
index c1d95911c..3fc16afef 100644
--- a/.gitignorerevs
+++ b/.gitignorerevs
@@ -1,2 +1,4 @@
# Run entire repository through black. Also, change tabs to spaces
3812225ace5db8c691cbe027d234a3e223972199
+# black 21.12b0 reformatting
+a5092f11f31b6a9d54a34423088a7bb8e70bc1c6
diff --git a/DEVELOPING b/DEVELOPING
index 7bc559125..0299b0364 100644
--- a/DEVELOPING
+++ b/DEVELOPING
@@ -224,20 +224,25 @@ Releases
First update the NEWS and RELEASE-NOTES files and commit.
Second create a git tag for this release:
- git tag v2.2.8
+ git tag portage-2.2.8
-Then create the tarball and run the tests:
- ./mkrelease.sh --changelog-rev v2.2.7 --tag --runtests 2.2.8
-Make sure you have all supported python versions installed first
+Create the tarball and run the tests: ./runtests
(see PYTHON_SUPPORTED_VERSIONS in runtests).
-Version bump the ebuild and verify it can re-install itself:
- emerge portage
- emerge portage
+Version bump the ebuild locally (don't push) and verify it can re-install itself:
+ emerge --oneshot portage
+ emerge --oneshot portage # this is using the new Portage
Publish the results (no going back now):
- Push the new git tag
- Upload the tarball
- Commit the new ebuild version
+Create the release for pypi and upload it there:
+ - python -m venv .venv
+ - . .venv/bin/activate
+ - pip install wheel twine
+ - python setup.py bdist_wheel sdist
+ - twine upload dist/<filenames>
+
Close the bugs blocking the tracker bug for this release.
diff --git a/MANIFEST.in b/MANIFEST.in
index 7d9ea761a..31ca9c166 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -25,3 +25,6 @@ include misc/*
# extensions
include src/*
+
+# GPG test keys
+recursive-include lib/portage/tests/.gnupg *
diff --git a/NEWS b/NEWS
index 1e772fdfd..da4d2d040 100644
--- a/NEWS
+++ b/NEWS
@@ -1,5 +1,20 @@
News (mainly features/major bug fixes)
+portage-3.0.30
+--------------
+* fix warning/crash on bad PROVIDES in package. See bug 828966.
+* avoid infinite loop in _optimized_copyfile. See bug 828844.
+
+portage-3.0.29
+--------------
+* check for inconsistent/missing PROVIDES during package (and binpkg)
+ installation. See bug 811462.
+* avoid pulling in newer slots for upgrades in some cases. See bug 828136.
+* enable SONAME dependencies for binpkgs by default. See bug 687956.
+* add new options to allow avoiding building/using binpkgs for live ebuilds:
+ FEATURES="binpkg-live" and --usepkg-exclude-live.
+* add emphasis in output to user patches being applied.
+
portage-3.0.27
--------------
* improve blocker output to make it clear which blockers
diff --git a/RELEASE-NOTES b/RELEASE-NOTES
index 937361db1..f9e4a9854 100644
--- a/RELEASE-NOTES
+++ b/RELEASE-NOTES
@@ -1,6 +1,15 @@
Release Notes; upgrade information mainly.
Features/major bugfixes are listed in NEWS
+portage-3.0.29
+==================================
+ - Bug 811462: fixes to detect package (and binpkg) corruption due to broken
+ scanelf.
+
+portage-3.0.28
+==================================
+ - Bug 816291: revert FILESDIR to symlink to fix SELinux regression.
+
portage-3.0.27
==================================
- Bug 815871: fix creating FILESDIR when FEATURES=keepwork is used
diff --git a/bin/ebuild-ipc.py b/bin/ebuild-ipc.py
index 4999c043a..c24ba6f58 100755
--- a/bin/ebuild-ipc.py
+++ b/bin/ebuild-ipc.py
@@ -138,9 +138,9 @@ class EbuildIpc:
def __init__(self):
self.fifo_dir = os.environ["PORTAGE_BUILDDIR"]
- self.ipc_in_fifo = os.path.join(self.fifo_dir, ".ipc_in")
- self.ipc_out_fifo = os.path.join(self.fifo_dir, ".ipc_out")
- self.ipc_lock_file = os.path.join(self.fifo_dir, ".ipc_lock")
+ self.ipc_in_fifo = os.path.join(self.fifo_dir, ".ipc", "in")
+ self.ipc_out_fifo = os.path.join(self.fifo_dir, ".ipc", "out")
+ self.ipc_lock_file = os.path.join(self.fifo_dir, ".ipc", "lock")
def _daemon_is_alive(self):
try:
diff --git a/bin/ebuild.sh b/bin/ebuild.sh
index 07ca58d22..ba406034b 100755
--- a/bin/ebuild.sh
+++ b/bin/ebuild.sh
@@ -198,45 +198,6 @@ fi
#(not secretive, but not stupid)
umask 022
-# debug-print() gets called from many places with verbose status information useful
-# for tracking down problems. The output is in $T/eclass-debug.log.
-# You can set ECLASS_DEBUG_OUTPUT to redirect the output somewhere else as well.
-# The special "on" setting echoes the information, mixing it with the rest of the
-# emerge output.
-# You can override the setting by exporting a new one from the console, or you can
-# set a new default in make.*. Here the default is "" or unset.
-
-# in the future might use e* from /etc/init.d/functions.sh if i feel like it
-debug-print() {
- # if $T isn't defined, we're in dep calculation mode and
- # shouldn't do anything
- [[ $EBUILD_PHASE = depend || ! -d ${T} || ${#} -eq 0 ]] && return 0
-
- if [[ ${ECLASS_DEBUG_OUTPUT} == on ]]; then
- printf 'debug: %s\n' "${@}" >&2
- elif [[ -n ${ECLASS_DEBUG_OUTPUT} ]]; then
- printf 'debug: %s\n' "${@}" >> "${ECLASS_DEBUG_OUTPUT}"
- fi
-
- if [[ -w $T ]] ; then
- # default target
- printf '%s\n' "${@}" >> "${T}/eclass-debug.log"
- # let the portage user own/write to this file
- chgrp "${PORTAGE_GRPNAME:-portage}" "${T}/eclass-debug.log"
- chmod g+w "${T}/eclass-debug.log"
- fi
-}
-
-# The following 2 functions are debug-print() wrappers
-
-debug-print-function() {
- debug-print "${1}: entering function, parameters: ${*:2}"
-}
-
-debug-print-section() {
- debug-print "now in section ${*}"
-}
-
# Sources all eclasses in parameters
declare -ix ECLASS_DEPTH=0
inherit() {
diff --git a/bin/estrip b/bin/estrip
index 665f377fd..599ca2ceb 100755
--- a/bin/estrip
+++ b/bin/estrip
@@ -172,7 +172,7 @@ save_elf_sources() {
if ! ${debugedit_warned} ; then
debugedit_warned=true
ewarn "FEATURES=installsources is enabled but the debugedit binary could not be"
- ewarn "found. This feature will not work unless debugedit or rpm is installed!"
+ ewarn "found. This feature will not work unless debugedit is installed!"
fi
return 0
fi
@@ -189,66 +189,90 @@ save_elf_sources() {
"${x}")
}
-# Usage: save_elf_debug <elf> [splitdebug file]
+# Usage: save_elf_debug <src> <inode_debug> [splitdebug]
save_elf_debug() {
${FEATURES_splitdebug} || return 0
${PORTAGE_RESTRICT_splitdebug} && return 0
+ debug-print-function "${FUNCNAME}" "$@"
+
# NOTE: Debug files must be installed in
# ${EPREFIX}/usr/lib/debug/${EPREFIX} (note that ${EPREFIX} occurs
# twice in this path) in order for gdb's debug-file-directory
# lookup to work correctly.
- local x=$1
- local inode_debug=$2
- local splitdebug=$3
- local d_noslash=${D%/}
- local y=${ED%/}/usr/lib/debug/${x:${#d_noslash}}.debug
+ local src=$1 # File from which we extract symbols.
+ local inode_debug=$2 # Temp path for hard link tracking
+ local splitdebug=$3 # Existing debug file optionally created by eu-strip in parent function
- # dont save debug info twice
- [[ ${x} == *".debug" ]] && return 0
+ # Source paths
+ local src_basename=${src##*/}
+ local src_dirname=${src%/*}
- mkdir -p "${y%/*}"
+ # Destination paths
+ local dst_dirname=${ED%/}/usr/lib/debug/${src_dirname#${D%/}/}
+ local dst_basename dst
- if [ -f "${inode_debug}" ] ; then
- ln "${inode_debug}" "${y}" || die "ln failed unexpectedly"
+ # dont save debug info twice
+ [[ ${src} == *".debug" ]] && return 0
+
+ mkdir -p "${dst_dirname}" || die "failed to create directory '${dst_dirname}'"
+
+ if [[ -L ${inode_debug} ]] ; then
+ # We already created a debug file for this inode.
+ # Read back the file name, and create another hard link if necessary.
+ dst_basename=$(readlink "${inode_debug}") || die "failed to read link '${inode_debug}'"
+ dst_basename=${dst_basename##*/}
+ dst=${dst_dirname}/${dst_basename}
+ if [[ ! -e ${dst} ]]; then
+ debug-print "creating hard link: target: '${inode_debug}' name: '${dst}'"
+ ln -L "${inode_debug}" "${dst}" || die "failed to create hard link '${dst}'"
+ fi
else
+ dst_basename=${src_basename}.debug
+ dst=${dst_dirname}/${dst_basename}
if [[ -n ${splitdebug} ]] ; then
- mv "${splitdebug}" "${y}"
+ mv "${splitdebug}" "${dst}"
else
local objcopy_flags="--only-keep-debug"
${FEATURES_compressdebug} && objcopy_flags+=" --compress-debug-sections"
- ${OBJCOPY} ${objcopy_flags} "${x}" "${y}"
- ${OBJCOPY} --add-gnu-debuglink="${y}" "${x}"
+ ${OBJCOPY} ${objcopy_flags} "${src}" "${dst}" &&
+ ${OBJCOPY} --add-gnu-debuglink="${dst}" "${src}"
fi
# Only do the following if the debug file was
# successfully created (see bug #446774).
- if [ $? -eq 0 ] ; then
+ if [[ $? -eq 0 ]] ; then
local args="a-x,o-w"
- [[ -g ${x} || -u ${x} ]] && args+=",go-r"
- chmod ${args} "${y}"
- ln "${y}" "${inode_debug}" || die "ln failed unexpectedly"
+ [[ -g ${src} || -u ${src} ]] && args+=",go-r"
+ chmod ${args} "${dst}"
+ # symlink so we can read the name back.
+ ln -s "${dst}" "${inode_debug}" || die "failed to create symlink '${inode_debug}'"
+
+ # if we don't already have build-id from debugedit, look it up
+ if [[ -z ${buildid} ]] ; then
+ # convert the readelf output to something useful
+ buildid=$(${READELF} -n "${src}" 2>/dev/null | awk '/Build ID:/{ print $NF; exit }')
+ fi
+ if [[ -n ${buildid} ]] ; then
+ local buildid_dir="${ED%/}/usr/lib/debug/.build-id/${buildid:0:2}"
+ local buildid_file="${buildid_dir}/${buildid:2}"
+ local src_buildid_rel="../../../../../${src#${ED%/}/}"
+ local dst_buildid_rel="../../${dst#${ED%/}/usr/lib/debug/}"
+ mkdir -p "${buildid_dir}" || die
+ [[ -L "${buildid_file}".debug ]] || ln -s "${dst_buildid_rel}" "${buildid_file}.debug" || die "failed to create symlink '${buildid_file}.debug'"
+ [[ -L "${buildid_file}" ]] || ln -s "${src_buildid_rel}" "${buildid_file}" || die "failed to create symlink '${buildid_file}'"
+ fi
fi
fi
- # if we don't already have build-id from debugedit, look it up
- if [[ -z ${buildid} ]] ; then
- # convert the readelf output to something useful
- buildid=$(${READELF} -n "${x}" 2>/dev/null | awk '/Build ID:/{ print $NF; exit }')
- fi
- if [[ -n ${buildid} ]] ; then
- local buildid_dir="${ED%/}/usr/lib/debug/.build-id/${buildid:0:2}"
- local buildid_file="${buildid_dir}/${buildid:2}"
- mkdir -p "${buildid_dir}"
- [ -L "${buildid_file}".debug ] || ln -s "../../${x:$((${#d_noslash} + 1))}.debug" "${buildid_file}.debug"
- [ -L "${buildid_file}" ] || ln -s "/${x:$((${#d_noslash} + 1))}" "${buildid_file}"
- fi
}
# Usage: process_elf <elf>
process_elf() {
local x=$1 inode_link=$2 strip_flags=${*:3}
local ed_noslash=${ED%/}
- local already_stripped lockfile xt_data
+ local already_stripped xt_data
+ local lockfile=${inode_link}_lockfile
+ local locktries=100
__vecho " ${x:${#ed_noslash}}"
@@ -257,13 +281,10 @@ process_elf() {
# So, use a lockfile to prevent interference (easily observed with
# dev-vcs/git which creates ~111 hardlinks to one file in
# /usr/libexec/git-core).
- lockfile=${inode_link}_lockfile
- if ! ln "${inode_link}" "${lockfile}" 2>/dev/null ; then
- while [[ -f ${lockfile} ]] ; do
- sleep 1
- done
- unset lockfile
- fi
+ while ! ln "${inode_link}" "${lockfile}" 2>/dev/null; do
+ (( --locktries > 0 )) || die "failed to acquire lock '${lockfile}'"
+ sleep 1
+ done
[ -f "${inode_link}_stripped" ] && already_stripped=true || already_stripped=false
diff --git a/bin/gpkg-helper.py b/bin/gpkg-helper.py
new file mode 100755
index 000000000..d79430212
--- /dev/null
+++ b/bin/gpkg-helper.py
@@ -0,0 +1,71 @@
+#!/usr/bin/python -b
+# Copyright 2020 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import argparse
+import sys
+import portage
+
+portage._internal_caller = True
+from portage import os
+
+
+def command_compose(args):
+
+ usage = "usage: compose <package_cpv> <binpkg_path> <metadata_dir> <image_dir>\n"
+
+ if len(args) != 4:
+ sys.stderr.write(usage)
+ sys.stderr.write("4 arguments are required, got %s\n" % len(args))
+ return 1
+
+ cpv, binpkg_path, metadata_dir, image_dir = args
+
+ if not os.path.isdir(metadata_dir):
+ sys.stderr.write(usage)
+ sys.stderr.write("Argument 3 is not a directory: '%s'\n" % metadata_dir)
+ return 1
+
+ if not os.path.isdir(image_dir):
+ sys.stderr.write(usage)
+ sys.stderr.write("Argument 4 is not a directory: '%s'\n" % image_dir)
+ return 1
+
+ gpkg_file = portage.gpkg.gpkg(portage.settings, cpv, binpkg_path)
+ metadata = gpkg_file._generate_metadata_from_dir(metadata_dir)
+ gpkg_file.compress(image_dir, metadata)
+ return os.EX_OK
+
+
+def main(argv):
+
+ if argv and isinstance(argv[0], bytes):
+ for i, x in enumerate(argv):
+ argv[i] = portage._unicode_decode(x, errors="strict")
+
+ valid_commands = ("compress",)
+ description = "Perform metadata operations on a binary package."
+ usage = "usage: %s COMMAND [args]" % os.path.basename(argv[0])
+
+ parser = argparse.ArgumentParser(description=description, usage=usage)
+ options, args = parser.parse_known_args(argv[1:])
+
+ if not args:
+ parser.error("missing command argument")
+
+ command = args[0]
+
+ if command not in valid_commands:
+ parser.error("invalid command: '%s'" % command)
+
+ if command == "compress":
+ rval = command_compose(args[1:])
+ else:
+ raise AssertionError("invalid command: '%s'" % command)
+
+ return rval
+
+
+if __name__ == "__main__":
+ rval = main(sys.argv[:])
+ sys.exit(rval)
diff --git a/bin/isolated-functions.sh b/bin/isolated-functions.sh
index df806d403..6d9b73474 100644
--- a/bin/isolated-functions.sh
+++ b/bin/isolated-functions.sh
@@ -609,4 +609,43 @@ else
}
fi
+# debug-print() gets called from many places with verbose status information useful
+# for tracking down problems. The output is in $T/eclass-debug.log.
+# You can set ECLASS_DEBUG_OUTPUT to redirect the output somewhere else as well.
+# The special "on" setting echoes the information, mixing it with the rest of the
+# emerge output.
+# You can override the setting by exporting a new one from the console, or you can
+# set a new default in make.*. Here the default is "" or unset.
+
+# in the future might use e* from /etc/init.d/functions.sh if i feel like it
+debug-print() {
+ # if $T isn't defined, we're in dep calculation mode and
+ # shouldn't do anything
+ [[ $EBUILD_PHASE = depend || ! -d ${T} || ${#} -eq 0 ]] && return 0
+
+ if [[ ${ECLASS_DEBUG_OUTPUT} == on ]]; then
+ printf 'debug: %s\n' "${@}" >&2
+ elif [[ -n ${ECLASS_DEBUG_OUTPUT} ]]; then
+ printf 'debug: %s\n' "${@}" >> "${ECLASS_DEBUG_OUTPUT}"
+ fi
+
+ if [[ -w $T ]] ; then
+ # default target
+ printf '%s\n' "${@}" >> "${T}/eclass-debug.log"
+ # let the portage user own/write to this file
+ chgrp "${PORTAGE_GRPNAME:-portage}" "${T}/eclass-debug.log"
+ chmod g+w "${T}/eclass-debug.log"
+ fi
+}
+
+# The following 2 functions are debug-print() wrappers
+
+debug-print-function() {
+ debug-print "${1}: entering function, parameters: ${*:2}"
+}
+
+debug-print-section() {
+ debug-print "now in section ${*}"
+}
+
true
diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh
index e4defa550..ccb07075f 100755
--- a/bin/misc-functions.sh
+++ b/bin/misc-functions.sh
@@ -503,38 +503,53 @@ __dyn_package() {
# in there in case any tools were built with -pg in CFLAGS.
cd "${T}" || die
- local tar_options=""
- [[ $PORTAGE_VERBOSE = 1 ]] && tar_options+=" -v"
- has xattr ${FEATURES} && [[ $(tar --help 2> /dev/null) == *--xattrs* ]] && tar_options+=" --xattrs"
# Sandbox is disabled in case the user wants to use a symlink
# for $PKGDIR and/or $PKGDIR/All.
export SANDBOX_ON="0"
- [ -z "${PORTAGE_BINPKG_TMPFILE}" ] && \
+ [[ -z "${PORTAGE_BINPKG_TMPFILE}" ]] && \
die "PORTAGE_BINPKG_TMPFILE is unset"
mkdir -p "${PORTAGE_BINPKG_TMPFILE%/*}" || die "mkdir failed"
- [ -z "${PORTAGE_COMPRESSION_COMMAND}" ] && \
- die "PORTAGE_COMPRESSION_COMMAND is unset"
- tar $tar_options -cf - $PORTAGE_BINPKG_TAR_OPTS -C "${D}" . | \
- $PORTAGE_COMPRESSION_COMMAND > "$PORTAGE_BINPKG_TMPFILE"
- assert "failed to pack binary package: '$PORTAGE_BINPKG_TMPFILE'"
- PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \
- "${PORTAGE_PYTHON:-/usr/bin/python}" "$PORTAGE_BIN_PATH"/xpak-helper.py recompose \
- "$PORTAGE_BINPKG_TMPFILE" "$PORTAGE_BUILDDIR/build-info"
- if [ $? -ne 0 ]; then
- rm -f "${PORTAGE_BINPKG_TMPFILE}"
- die "Failed to append metadata to the tbz2 file"
- fi
- local md5_hash=""
- if type md5sum &>/dev/null ; then
- md5_hash=$(md5sum "${PORTAGE_BINPKG_TMPFILE}")
- md5_hash=${md5_hash%% *}
- elif type md5 &>/dev/null ; then
- md5_hash=$(md5 "${PORTAGE_BINPKG_TMPFILE}")
- md5_hash=${md5_hash##* }
+
+ if [[ "${BINPKG_FORMAT}" == "xpak" ]]; then
+ local tar_options=""
+ [[ $PORTAGE_VERBOSE = 1 ]] && tar_options+=" -v"
+ has xattr ${FEATURES} && [[ $(tar --help 2> /dev/null) == *--xattrs* ]] && tar_options+=" --xattrs"
+ [[ -z "${PORTAGE_COMPRESSION_COMMAND}" ]] && \
+ die "PORTAGE_COMPRESSION_COMMAND is unset"
+ tar $tar_options -cf - $PORTAGE_BINPKG_TAR_OPTS -C "${D}" . | \
+ $PORTAGE_COMPRESSION_COMMAND > "$PORTAGE_BINPKG_TMPFILE"
+ assert "failed to pack binary package: '$PORTAGE_BINPKG_TMPFILE'"
+ PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \
+ "${PORTAGE_PYTHON:-/usr/bin/python}" "$PORTAGE_BIN_PATH"/xpak-helper.py recompose \
+ "$PORTAGE_BINPKG_TMPFILE" "$PORTAGE_BUILDDIR/build-info"
+ if [[ $? -ne 0 ]]; then
+ rm -f "${PORTAGE_BINPKG_TMPFILE}"
+ die "Failed to append metadata to the tbz2 file"
+ fi
+ local md5_hash=""
+ if type md5sum &>/dev/null ; then
+ md5_hash=$(md5sum "${PORTAGE_BINPKG_TMPFILE}")
+ md5_hash=${md5_hash%% *}
+ elif type md5 &>/dev/null ; then
+ md5_hash=$(md5 "${PORTAGE_BINPKG_TMPFILE}")
+ md5_hash=${md5_hash##* }
+ fi
+ [[ -n "${md5_hash}" ]] && \
+ echo ${md5_hash} > "${PORTAGE_BUILDDIR}"/build-info/BINPKGMD5
+ __vecho ">>> Done."
+
+ elif [[ "${BINPKG_FORMAT}" == "gpkg" ]]; then
+ PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \
+ "${PORTAGE_PYTHON:-/usr/bin/python}" "$PORTAGE_BIN_PATH"/gpkg-helper.py compress \
+ "${CATEGORY}/${PF}" "$PORTAGE_BINPKG_TMPFILE" "$PORTAGE_BUILDDIR/build-info" "${D}"
+ if [[ $? -ne 0 ]]; then
+ rm -f "${PORTAGE_BINPKG_TMPFILE}"
+ die "Failed to create binpkg file"
+ fi
+ __vecho ">>> Done."
+ else
+ die "Unknown BINPKG_FORMAT ${BINPKG_FORMAT}"
fi
- [ -n "${md5_hash}" ] && \
- echo ${md5_hash} > "${PORTAGE_BUILDDIR}"/build-info/BINPKGMD5
- __vecho ">>> Done."
cd "${PORTAGE_BUILDDIR}"
>> "$PORTAGE_BUILDDIR/.packaged" || \
diff --git a/bin/phase-functions.sh b/bin/phase-functions.sh
index d3221993d..5eb031805 100644
--- a/bin/phase-functions.sh
+++ b/bin/phase-functions.sh
@@ -291,10 +291,10 @@ __dyn_clean() {
rm -f "$PORTAGE_BUILDDIR"/.{ebuild_changed,logid,pretended,setuped,unpacked,prepared} \
"$PORTAGE_BUILDDIR"/.{configured,compiled,tested,packaged,instprepped} \
"$PORTAGE_BUILDDIR"/.die_hooks \
- "$PORTAGE_BUILDDIR"/.ipc_{in,out,lock} \
"$PORTAGE_BUILDDIR"/.exit_status
- rm -rf "${PORTAGE_BUILDDIR}/build-info"
+ rm -rf "${PORTAGE_BUILDDIR}/build-info" \
+ "${PORTAGE_BUILDDIR}/.ipc"
rm -rf "${WORKDIR}"
rm -f "${PORTAGE_BUILDDIR}/files"
fi
diff --git a/bin/phase-helpers.sh b/bin/phase-helpers.sh
index 94f4f24f2..6a078caad 100644
--- a/bin/phase-helpers.sh
+++ b/bin/phase-helpers.sh
@@ -1139,6 +1139,10 @@ if ___eapi_has_eapply_user; then
local basedir=${PORTAGE_CONFIGROOT%/}/etc/portage/patches
+ local columns=${COLUMNS:-0}
+ [[ ${columns} == 0 ]] && columns=$(set -- $( ( stty size </dev/tty ) 2>/dev/null || echo 24 80 ) ; echo $2)
+ (( columns > 0 )) || (( columns = 80 ))
+
local applied d f
local -A _eapply_user_patches
local prev_shopt=$(shopt -p nullglob)
@@ -1170,6 +1174,11 @@ if ___eapi_has_eapply_user; then
while read -r -d '' f; do
f=${_eapply_user_patches[${f}]}
if [[ -s ${f} ]]; then
+ if [[ -z ${applied} ]]; then
+ einfo "${PORTAGE_COLOR_INFO}$(for ((column = 0; column < ${columns} - 3; column++)); do echo -n =; done)${PORTAGE_COLOR_NORMAL}"
+ einfo "Applying user patches from ${basedir} ..."
+ fi
+
eapply "${f}"
applied=1
fi
@@ -1179,7 +1188,10 @@ if ___eapi_has_eapply_user; then
${prev_shopt}
- [[ -n ${applied} ]] && ewarn "User patches applied."
+ if [[ -n ${applied} ]]; then
+ einfo "User patches applied."
+ einfo "${PORTAGE_COLOR_INFO}$(for ((column = 0; column < ${columns} - 3; column++)); do echo -n =; done)${PORTAGE_COLOR_NORMAL}"
+ fi
}
fi
diff --git a/bin/quickpkg b/bin/quickpkg
index 1b7ad666c..27dda85ed 100755
--- a/bin/quickpkg
+++ b/bin/quickpkg
@@ -15,17 +15,20 @@ if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), ".porta
import portage
portage._internal_caller = True
from portage import os
-from portage import xpak
+from portage import xpak, gpkg
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.dbapi.dep_expand import dep_expand
from portage.dep import Atom, use_reduce
from portage.exception import (AmbiguousPackageName, InvalidAtom, InvalidData,
- InvalidDependString, PackageSetNotFound, PermissionDenied)
+ InvalidBinaryPackageFormat, InvalidDependString, PackageSetNotFound,
+ PermissionDenied)
from portage.util import ensure_dirs, shlex_split, varexpand, _xattr
xattr = _xattr.xattr
from portage._sets import load_default_config, SETPREFIX
from portage.process import find_binary
from portage.util.compression_probe import _compressors
from portage.util._eventloop.global_event_loop import global_event_loop
+from portage.gpg import GPG
def quickpkg_atom(options, infos, arg, eout):
@@ -109,52 +112,66 @@ def quickpkg_atom(options, infos, arg, eout):
update_metadata[k] = v
if update_metadata:
vardb.aux_update(cpv, update_metadata)
- xpdata = xpak.xpak(dblnk.dbdir)
- binpkg_tmpfile = os.path.join(bintree.pkgdir,
- cpv + ".tbz2." + str(portage.getpid()))
- ensure_dirs(os.path.dirname(binpkg_tmpfile))
- binpkg_compression = settings.get("BINPKG_COMPRESS", "bzip2")
- try:
- compression = _compressors[binpkg_compression]
- except KeyError as e:
- if binpkg_compression:
+
+ binpkg_format = settings.get("BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0])
+ if binpkg_format == "xpak":
+ xpdata = xpak.xpak(dblnk.dbdir)
+ binpkg_tmpfile = os.path.join(bintree.pkgdir,
+ cpv + ".tbz2." + str(portage.getpid()))
+ ensure_dirs(os.path.dirname(binpkg_tmpfile))
+ binpkg_compression = settings.get("BINPKG_COMPRESS", "bzip2")
+ try:
+ compression = _compressors[binpkg_compression]
+ except KeyError as e:
+ if binpkg_compression:
+ eout.eerror("Invalid or unsupported compression method: %s" % e.args[0])
+ return 1
+ # Empty BINPKG_COMPRESS disables compression.
+ binpkg_compression = 'none'
+ compression = {
+ 'compress': 'cat',
+ 'package': 'sys-apps/coreutils',
+ }
+ try:
+ compression_binary = shlex_split(varexpand(compression["compress"], mydict=settings))[0]
+ except IndexError as e:
eout.eerror("Invalid or unsupported compression method: %s" % e.args[0])
return 1
- # Empty BINPKG_COMPRESS disables compression.
- binpkg_compression = 'none'
- compression = {
- 'compress': 'cat',
- 'package': 'sys-apps/coreutils',
- }
- try:
- compression_binary = shlex_split(varexpand(compression["compress"], mydict=settings))[0]
- except IndexError as e:
- eout.eerror("Invalid or unsupported compression method: %s" % e.args[0])
- return 1
- if find_binary(compression_binary) is None:
- missing_package = compression["package"]
- eout.eerror("File compression unsupported %s. Missing package: %s" % (binpkg_compression, missing_package))
- return 1
- cmd = shlex_split(varexpand(compression["compress"], mydict=settings))
- # Filter empty elements that make Popen fail
- cmd = [x for x in cmd if x != ""]
- with open(binpkg_tmpfile, "wb") as fobj:
- proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=fobj)
- excluded_config_files = dblnk.quickpkg(proc.stdin,
+ if find_binary(compression_binary) is None:
+ missing_package = compression["package"]
+ eout.eerror("File compression unsupported %s. Missing package: %s" % (binpkg_compression, missing_package))
+ return 1
+ cmd = [varexpand(x, mydict=settings) for x in shlex_split(compression["compress"])]
+ # Filter empty elements that make Popen fail
+ cmd = [x for x in cmd if x != ""]
+ with open(binpkg_tmpfile, "wb") as fobj:
+ proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=fobj)
+ excluded_config_files = dblnk.quickpkg(proc.stdin,
+ include_config=include_config,
+ include_unmodified_config=include_unmodified_config)
+ proc.stdin.close()
+ if proc.wait() != os.EX_OK:
+ eout.eend(1)
+ eout.eerror("Compressor failed for package %s" % cpv)
+ retval |= 1
+ try:
+ os.unlink(binpkg_tmpfile)
+ except OSError as e:
+ if e.errno not in (errno.ENOENT, errno.ESTALE):
+ raise
+ continue
+ xpak.tbz2(binpkg_tmpfile).recompose_mem(xpdata)
+ elif binpkg_format == "gpkg":
+ metadata = gpkg.gpkg(settings)._generate_metadata_from_dir(dblnk.dbdir)
+ binpkg_tmpfile = os.path.join(bintree.pkgdir,
+ cpv + ".gpkg.tar." + str(os.getpid()))
+ ensure_dirs(os.path.dirname(binpkg_tmpfile))
+ excluded_config_files = dblnk.quickpkg(binpkg_tmpfile,
+ metadata,
include_config=include_config,
include_unmodified_config=include_unmodified_config)
- proc.stdin.close()
- if proc.wait() != os.EX_OK:
- eout.eend(1)
- eout.eerror("Compressor failed for package %s" % cpv)
- retval |= 1
- try:
- os.unlink(binpkg_tmpfile)
- except OSError as e:
- if e.errno not in (errno.ENOENT, errno.ESTALE):
- raise
- continue
- xpak.tbz2(binpkg_tmpfile).recompose_mem(xpdata)
+ else:
+ raise InvalidBinaryPackageFormat(binpkg_format)
finally:
if have_lock:
dblnk.unlockdb()
@@ -268,6 +285,10 @@ def quickpkg_main(options, args, eout):
portage.settings.features.remove('xattr')
portage.settings.lock()
+ if portage.settings.get("BINPKG_GPG_SIGNING_KEY", None):
+ gpg = GPG(portage.settings)
+ gpg.unlock()
+
infos = {}
infos["successes"] = []
infos["missing"] = []
diff --git a/cnf/make.conf.example b/cnf/make.conf.example
index a309a5c43..5b2229465 100644
--- a/cnf/make.conf.example
+++ b/cnf/make.conf.example
@@ -183,6 +183,42 @@
# This ftp connection is active ftp.
#PORTAGE_BINHOST="ftp://login:pass@grp.mirror.site:21*/pub/grp/i686/athlon-xp/"
+# Binary packages GPG commands
+# ============================
+#
+# Only works with GPKG format.
+# "binpkg-signing" needed to be set in FEATURES if need signing packages.
+# "binpkg-request-signature" needed to be set in FEATURES if you want all
+# binpkgs must have signature.
+# You need uncomment related commands and set "USER" and "SIGN_KEY" to yours.
+#
+# Binary package GPG singing base command
+# Basic command for all signature operations.
+# You need change this if you want to use other configurations,
+# Note that some configurations are configured separately below,
+# please do not add duplicate configurations
+#BINPKG_GPG_SIGNING_BASE_COMMAND="/usr/bin/flock /run/lock/portage-binpkg-gpg.lock /usr/bin/gpg --sign --armor [PORTAGE_CONFIG]"
+
+# Binary package GPG signature digests algorithm.
+#BINPKG_GPG_SIGNING_DIGEST="SHA512"
+
+# gnupg home directory used for signing.
+#BINPKG_GPG_SIGNING_GPG_HOME="/root/.gnupg"
+
+# GPG key ID used for signing.
+#BINPKG_GPG_SIGNING_KEY="0x1234567890ABCD!"
+
+# Binary package GPG verify base command.
+# Basic command for all verify operations.
+#BINPKG_GPG_VERIFY_BASE_COMMAND="/usr/bin/gpg --verify --batch --no-tty --no-auto-check-trustdb --status-fd 2 [PORTAGE_CONFIG] [SIGNATURE]"
+
+# GPG home directory where store all trust binary package public keys.
+#BINPKG_GPG_VERIFY_GPG_HOME="/etc/portage/gnupg"
+
+# The user and group will be used when drop root privileges during GPG verify
+#GPG_VERIFY_USER_DROP="nobody"
+#GPG_VERIFY_GROUP_DROP="nogroup"
+
# Synchronizing Portage
# =====================
#
diff --git a/cnf/make.conf.example.riscv.diff b/cnf/make.conf.example.riscv.diff
new file mode 100644
index 000000000..df1e6a70d
--- /dev/null
+++ b/cnf/make.conf.example.riscv.diff
@@ -0,0 +1,61 @@
+--- make.conf.example
++++ make.conf.example
+@@ -22,6 +22,15 @@
+ # Example:
+ #USE="X gtk gnome -alsa"
+
++# Host Setting
++# ===========
++#
++# All RV64 systems should use this host setting:
++CHOST="riscv64-unknown-linux-gnu"
++
++# whereas all RV32 ones should use:
++#CHOST="riscv32-unknown-linux-gnu"
++
+ # Host and optimization settings
+ # ==============================
+ #
+@@ -39,9 +48,31 @@
+ # -frecord-gcc-switches, since otherwise the check could result in false
+ # positive results.
+ #
+-# Please refer to the GCC manual for a list of possible values.
++# -mcpu=<cpu-type> tells the compiler to take full advantage of the ABI and
++# instructions available on a certain CPU. This will produce code which may
++# not run on other RISC-V CPUs, even if they support the same ISA.
++#
++# -mtune=<cpu-type-or-series> results in code optimised for a specific CPU
++# (e.g. SiFive U74) or family (e.g. the SiFive 7 series) without breaking
++# compatibility with other RISC-V CPUs supporting the same ISA.
++#
++# -march=<isa-string> specifies the exact ISA for which to generate code.
++# This is normally not needed if you set -mcpu but might be useful for CPUs
++# not explicitly supported by gcc yet. Should that be the case and you aren't
++# sure what ISA string to use, the output of /proc/cpuinfo might give you a hint.
++#
++# -mabi=<abi-string> specifies the integer and floating-point calling convention
++# to use. Again, this is normally not needed if you set -mcpu. Care should be
++# taken while setting both -march and -mabi, as some calling conventions are
++# impossible to implement on some ISAs.
++#
++# Please refer to the section "RISC-V Options" of the GCC manual for a list
++# of possible ABI, architecture, CPU and feature values.
++#
++# Decent examples:
+ #
+-#CFLAGS="-O2 -pipe"
++#CFLAGS="-mcpu=sifive-u74 -O2 -pipe"
++#CFLAGS="-mtune=sifive-7-series -O2 -pipe"
+ #
+ # If you set a CFLAGS above, then this line will set your default C++ flags to
+ # the same settings.
+@@ -76,7 +107,7 @@
+ # DO NOT PUT ANYTHING BUT YOUR SPECIFIC ~ARCHITECTURE IN THE LIST.
+ # IF YOU ARE UNSURE OF YOUR ARCH, OR THE IMPLICATIONS, DO NOT MODIFY THIS.
+ #
+-#ACCEPT_KEYWORDS="~arch"
++#ACCEPT_KEYWORDS="~riscv"
+
+ # ACCEPT_LICENSE is used to mask packages based on licensing restrictions.
+ # It may contain both license and group names, where group names are
diff --git a/cnf/make.globals b/cnf/make.globals
index b8bc90ae0..69b365f71 100644
--- a/cnf/make.globals
+++ b/cnf/make.globals
@@ -38,6 +38,27 @@ PORTAGE_TMPDIR="/var/tmp"
# existing installs where bzip2 is used for backward compatibility.
BINPKG_COMPRESS="zstd"
+# The format used for binary packages. The default is use old "xpak" format.
+# Set to "gpkg" to use new gentoo binary package format.
+BINPKG_FORMAT="xpak"
+
+# The binary package default GPG signing command.
+# flock is used to avoid a racing condition of gnupg
+BINPKG_GPG_SIGNING_BASE_COMMAND="/usr/bin/flock /run/lock/portage-binpkg-gpg.lock /usr/bin/gpg --sign --armor [PORTAGE_CONFIG]"
+
+# The default binary package GPG digests algorithm.
+BINPKG_GPG_SIGNING_DIGEST="SHA512"
+
+# The binary package default GPG verify command.
+BINPKG_GPG_VERIFY_BASE_COMMAND="/usr/bin/gpg --verify --batch --no-tty --no-auto-check-trustdb --status-fd 2 [PORTAGE_CONFIG] [SIGNATURE]"
+
+# The binary package default GPG home directory for verify
+BINPKG_GPG_VERIFY_GPG_HOME="/etc/portage/gnupg"
+
+# The user and group will be used when drop root privileges during GPG verify
+GPG_VERIFY_USER_DROP="nobody"
+GPG_VERIFY_GROUP_DROP="nogroup"
+
# Fetching command (3 tries, passive ftp for firewall compatibility)
FETCHCOMMAND="wget -t 3 -T 60 --passive-ftp -O \"\${DISTDIR}/\${FILE}\" \"\${URI}\""
RESUMECOMMAND="wget -c -t 3 -T 60 --passive-ftp -O \"\${DISTDIR}/\${FILE}\" \"\${URI}\""
@@ -54,7 +75,7 @@ FETCHCOMMAND_SFTP="bash -c \"x=\\\${2#sftp://} ; host=\\\${x%%/*} ; port=\\\${ho
# Default user options
FEATURES="assume-digests binpkg-docompress binpkg-dostrip binpkg-logs
- binpkg-multi-instance
+ binpkg-multi-instance buildpkg-live
config-protect-if-modified distlocks ebuild-locks
fixlafiles ipc-sandbox merge-sync multilib-strict
network-sandbox news parallel-fetch pid-sandbox
diff --git a/lib/_emerge/AbstractEbuildProcess.py b/lib/_emerge/AbstractEbuildProcess.py
index 1b4e7759f..6d89d40f0 100644
--- a/lib/_emerge/AbstractEbuildProcess.py
+++ b/lib/_emerge/AbstractEbuildProcess.py
@@ -249,8 +249,8 @@ class AbstractEbuildProcess(SpawnProcess):
def _init_ipc_fifos(self):
- input_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".ipc_in")
- output_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".ipc_out")
+ input_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".ipc", "in")
+ output_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".ipc", "out")
for p in (input_fifo, output_fifo):
diff --git a/lib/_emerge/Binpkg.py b/lib/_emerge/Binpkg.py
index c7dde69bd..15eb56092 100644
--- a/lib/_emerge/Binpkg.py
+++ b/lib/_emerge/Binpkg.py
@@ -2,7 +2,6 @@
# Distributed under the terms of the GNU General Public License v2
import functools
-
import _emerge.emergelog
from _emerge.EbuildPhase import EbuildPhase
from _emerge.BinpkgFetcher import BinpkgFetcher
@@ -13,8 +12,11 @@ from _emerge.EbuildMerge import EbuildMerge
from _emerge.EbuildBuildDir import EbuildBuildDir
from _emerge.SpawnProcess import SpawnProcess
from portage.eapi import eapi_exports_replace_vars
+from portage.exception import PortageException
+from portage.output import colorize
from portage.util import ensure_dirs
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
+from portage.util._dyn_libs.dyn_libs import check_dyn_libs_inconsistent
import portage
from portage import os
from portage import shutil
@@ -416,15 +418,35 @@ class Binpkg(CompositeTask):
def _unpack_contents_exit(self, unpack_contents):
if self._default_exit(unpack_contents) != os.EX_OK:
- unpack_contents.future.result()
+ try:
+ unpack_contents.future.result()
+ err = ""
+ except PortageException as e:
+ err = e
+
self._writemsg_level(
- "!!! Error Extracting '%s'\n" % self._pkg_path,
+ colorize(
+ "BAD",
+ f"!!! Error Extracting '{self._pkg_path}', {err}\n",
+ ),
noiselevel=-1,
level=logging.ERROR,
)
self._async_unlock_builddir(returncode=self.returncode)
return
+ # Before anything else, let's do an integrity check.
+ (provides,) = self._bintree.dbapi.aux_get(self.pkg.cpv, ["PROVIDES"])
+ if check_dyn_libs_inconsistent(self.settings["D"], provides):
+ self._writemsg_level(
+ colorize(
+ "BAD",
+ "!!! Error! Installing dynamic libraries (.so) with blank PROVIDES!",
+ ),
+ noiselevel=-1,
+ level=logging.ERROR,
+ )
+
try:
with io.open(
_unicode_encode(
diff --git a/lib/_emerge/BinpkgExtractorAsync.py b/lib/_emerge/BinpkgExtractorAsync.py
index a0380a0a4..919837fc1 100644
--- a/lib/_emerge/BinpkgExtractorAsync.py
+++ b/lib/_emerge/BinpkgExtractorAsync.py
@@ -15,6 +15,8 @@ from portage.util import (
shlex_split,
varexpand,
)
+from portage.exception import InvalidBinaryPackageFormat
+from portage.binpkg import get_binpkg_format
import signal
import subprocess
import tarfile
@@ -27,6 +29,13 @@ class BinpkgExtractorAsync(SpawnProcess):
_shell_binary = portage.const.BASH_BINARY
def _start(self):
+ binpkg_format = get_binpkg_format(self.pkg_path)
+ if binpkg_format == "xpak":
+ self._xpak_start()
+ else:
+ raise InvalidBinaryPackageFormat(self.pkg_path)
+
+ def _xpak_start(self):
tar_options = ""
if "xattr" in self.features:
process = subprocess.Popen(
diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py
index de3dd42ed..d5275ea11 100644
--- a/lib/_emerge/BinpkgFetcher.py
+++ b/lib/_emerge/BinpkgFetcher.py
@@ -11,6 +11,8 @@ import stat
import sys
import portage
from portage import os
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
+from portage.exception import FileNotFound, InvalidBinaryPackageFormat
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
from portage.util._pty import _create_pty_or_pipe
@@ -21,8 +23,23 @@ class BinpkgFetcher(CompositeTask):
def __init__(self, **kwargs):
CompositeTask.__init__(self, **kwargs)
+
pkg = self.pkg
- self.pkg_path = pkg.root_config.trees["bintree"].getname(pkg.cpv) + ".partial"
+ bintree = pkg.root_config.trees["bintree"]
+ binpkg_path = None
+
+ if bintree._remote_has_index:
+ instance_key = bintree.dbapi._instance_key(pkg.cpv)
+ binpkg_path = bintree._remotepkgs[instance_key].get("PATH")
+ if binpkg_path:
+ self.pkg_path = binpkg_path + ".partial"
+ else:
+ self.pkg_path = (
+ pkg.root_config.trees["bintree"].getname(pkg.cpv, allocate_new=True)
+ + ".partial"
+ )
+ else:
+ raise FileNotFound("Binary packages index not found")
def _start(self):
fetcher = _BinpkgFetcherProcess(
@@ -106,15 +123,23 @@ class _BinpkgFetcherProcess(SpawnProcess):
resumecommand = None
if bintree._remote_has_index:
remote_metadata = bintree._remotepkgs[bintree.dbapi._instance_key(pkg.cpv)]
+ binpkg_format = remote_metadata.get(
+ "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
+ )
+ if binpkg_format not in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ raise InvalidBinaryPackageFormat(binpkg_format)
rel_uri = remote_metadata.get("PATH")
if not rel_uri:
- rel_uri = pkg.cpv + ".tbz2"
+ if binpkg_format == "xpak":
+ rel_uri = pkg.cpv + ".tbz2"
+ elif binpkg_format == "gpkg":
+ rel_uri = pkg.cpv + ".gpkg.tar"
remote_base_uri = remote_metadata["BASE_URI"]
uri = remote_base_uri.rstrip("/") + "/" + rel_uri.lstrip("/")
fetchcommand = remote_metadata.get("FETCHCOMMAND")
resumecommand = remote_metadata.get("RESUMECOMMAND")
else:
- uri = settings["PORTAGE_BINHOST"].rstrip("/") + "/" + pkg.pf + ".tbz2"
+ raise FileNotFound("Binary packages index not found")
if pretend:
portage.writemsg_stdout("\n%s\n" % uri, noiselevel=-1)
diff --git a/lib/_emerge/EbuildBinpkg.py b/lib/_emerge/EbuildBinpkg.py
index 5942d245a..ccdd30f7b 100644
--- a/lib/_emerge/EbuildBinpkg.py
+++ b/lib/_emerge/EbuildBinpkg.py
@@ -6,6 +6,8 @@ from _emerge.EbuildPhase import EbuildPhase
import portage
from portage import os
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
+from portage.exception import InvalidBinaryPackageFormat
class EbuildBinpkg(CompositeTask):
@@ -19,9 +21,19 @@ class EbuildBinpkg(CompositeTask):
pkg = self.pkg
root_config = pkg.root_config
bintree = root_config.trees["bintree"]
- binpkg_tmpfile = os.path.join(
- bintree.pkgdir, pkg.cpv + ".tbz2." + str(portage.getpid())
+ binpkg_format = self.settings.get(
+ "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
)
+ if binpkg_format == "xpak":
+ binpkg_tmpfile = os.path.join(
+ bintree.pkgdir, pkg.cpv + ".tbz2." + str(portage.getpid())
+ )
+ elif binpkg_format == "gpkg":
+ binpkg_tmpfile = os.path.join(
+ bintree.pkgdir, pkg.cpv + ".gpkg.tar." + str(portage.getpid())
+ )
+ else:
+ raise InvalidBinaryPackageFormat(binpkg_format)
bintree._ensure_dir(os.path.dirname(binpkg_tmpfile))
self._binpkg_tmpfile = binpkg_tmpfile
diff --git a/lib/_emerge/EbuildBuild.py b/lib/_emerge/EbuildBuild.py
index 2ed16537b..6d290e116 100644
--- a/lib/_emerge/EbuildBuild.py
+++ b/lib/_emerge/EbuildBuild.py
@@ -343,9 +343,24 @@ class EbuildBuild(CompositeTask):
and opts.buildpkg != "n"
)
+ # Do not build binary cache for packages from volatile sources.
+ # For volatile sources (eg., git), the PROPERTIES parameter in
+ # the ebuild is set to 'live'.
+
+ # The default behavior is to build binary cache for all pkgs.
+ # "buildpkg-live" is a FEATURE that is enabled by default.
+ # To not build binary cache for live pkgs, we disable it by
+ # specifying FEATURES="-buildpkg-live"
+
+ buildpkg_live = "buildpkg-live" in features
+ live_ebuild = "live" in self.settings.get("PROPERTIES", "").split()
+ buildpkg_live_disabled = live_ebuild and not buildpkg_live
+
if (
- "buildpkg" in features or self._issyspkg
- ) and not self.opts.buildpkg_exclude.findAtomForPackage(pkg):
+ ("buildpkg" in features or self._issyspkg)
+ and not buildpkg_live_disabled
+ and not self.opts.buildpkg_exclude.findAtomForPackage(pkg)
+ ):
self._buildpkg = True
diff --git a/lib/_emerge/EbuildIpcDaemon.py b/lib/_emerge/EbuildIpcDaemon.py
index ee6fd7658..78594ff0a 100644
--- a/lib/_emerge/EbuildIpcDaemon.py
+++ b/lib/_emerge/EbuildIpcDaemon.py
@@ -81,7 +81,7 @@ class EbuildIpcDaemon(FifoIpcDaemon):
# write something to the pipe just before we close it, and in that
# case the write will be lost. Therefore, try for a non-blocking
# lock, and only re-open the pipe if the lock is acquired.
- lock_filename = os.path.join(os.path.dirname(self.input_fifo), ".ipc_lock")
+ lock_filename = os.path.join(os.path.dirname(self.input_fifo), "lock")
try:
lock_obj = lockfile(lock_filename, unlinkfile=True, flags=os.O_NONBLOCK)
except TryAgain:
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index 12326fffd..9a04f9c1f 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -29,6 +29,8 @@ from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
from portage.util._async.BuildLogger import BuildLogger
from portage.util.futures import asyncio
from portage.util.futures.executor.fork import ForkExecutor
+from portage.exception import InvalidBinaryPackageFormat
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
try:
from portage.xml.metadata import MetaDataXML
@@ -157,14 +159,31 @@ class EbuildPhase(CompositeTask):
if self.phase == "package":
if "PORTAGE_BINPKG_TMPFILE" not in self.settings:
- self.settings["PORTAGE_BINPKG_TMPFILE"] = (
- os.path.join(
- self.settings["PKGDIR"],
- self.settings["CATEGORY"],
- self.settings["PF"],
- )
- + ".tbz2"
+ binpkg_format = self.settings.get(
+ "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
)
+ if binpkg_format == "xpak":
+ self.settings["BINPKG_FORMAT"] = "xpak"
+ self.settings["PORTAGE_BINPKG_TMPFILE"] = (
+ os.path.join(
+ self.settings["PKGDIR"],
+ self.settings["CATEGORY"],
+ self.settings["PF"],
+ )
+ + ".tbz2"
+ )
+ elif binpkg_format == "gpkg":
+ self.settings["BINPKG_FORMAT"] = "gpkg"
+ self.settings["PORTAGE_BINPKG_TMPFILE"] = (
+ os.path.join(
+ self.settings["PKGDIR"],
+ self.settings["CATEGORY"],
+ self.settings["PF"],
+ )
+ + ".gpkg.tar"
+ )
+ else:
+ raise InvalidBinaryPackageFormat(binpkg_format)
def _async_start_exit(self, task):
task.future.cancelled() or task.future.result()
diff --git a/lib/_emerge/Package.py b/lib/_emerge/Package.py
index 90dfccdef..cc2cb07e1 100644
--- a/lib/_emerge/Package.py
+++ b/lib/_emerge/Package.py
@@ -63,6 +63,7 @@ class Package(Task):
metadata_keys = [
"BDEPEND",
+ "BINPKG_FORMAT",
"BUILD_ID",
"BUILD_TIME",
"CHOST",
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 05a115250..0255e3e97 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -41,7 +41,7 @@ from portage.dbapi._expand_new_virt import expand_new_virt
from portage.dbapi.IndexedPortdb import IndexedPortdb
from portage.dbapi.IndexedVardb import IndexedVardb
from portage.dep import Atom, _repo_separator, _slot_separator
-from portage.exception import InvalidAtom, InvalidData, ParseError
+from portage.exception import InvalidAtom, InvalidData, ParseError, GPGException
from portage.output import (
colorize,
create_color_func,
@@ -77,6 +77,8 @@ from portage.sync.old_tree_timestamp import old_tree_timestamp_warn
from portage.localization import _
from portage.metadata import action_metadata
from portage.emaint.main import print_results
+from portage.gpg import GPG
+from portage.binpkg import get_binpkg_format
from _emerge.clear_caches import clear_caches
from _emerge.create_depgraph_params import create_depgraph_params
@@ -604,6 +606,28 @@ def action_build(
)
return 1
+ # unlock GPG if needed
+ if (
+ need_write_bindb
+ and (eroot in ebuild_eroots)
+ and (
+ "binpkg-signing"
+ in trees[eroot]["root_config"].settings.features
+ )
+ ):
+ portage.writemsg_stdout(">>> Unlocking GPG... ")
+ sys.stdout.flush()
+ gpg = GPG(trees[eroot]["root_config"].settings)
+ try:
+ gpg.unlock()
+ except GPGException as e:
+ writemsg_level(
+ colorize("BAD", "!!! %s\n" % e),
+ level=logging.ERROR,
+ noiselevel=-1,
+ )
+ return 1
+
if "--resume" in myopts:
favorites = mtimedb["resume"]["favorites"]
@@ -2271,11 +2295,21 @@ def action_info(settings, trees, myopts, myfiles):
elif pkg_type == "ebuild":
ebuildpath = portdb.findname(pkg.cpv, myrepo=pkg.repo)
elif pkg_type == "binary":
- tbz2_file = bindb.bintree.getname(pkg.cpv)
+ binpkg_file = bindb.bintree.getname(pkg.cpv)
ebuild_file_name = pkg.cpv.split("/")[1] + ".ebuild"
- ebuild_file_contents = portage.xpak.tbz2(tbz2_file).getfile(
- ebuild_file_name
- )
+ binpkg_format = pkg.cpv._metadata.get("BINPKG_FORMAT", None)
+ if not binpkg_format:
+ binpkg_format = get_binpkg_format(binpkg_file)
+ if binpkg_format == "xpak":
+ ebuild_file_contents = portage.xpak.tbz2(binpkg_file).getfile(
+ ebuild_file_name
+ )
+ elif binpkg_format == "gpkg":
+ ebuild_file_contents = portage.gpkg.gpkg(
+ settings, pkg.cpv, binpkg_file
+ ).get_metadata(ebuild_file_name)
+ else:
+ continue
tmpdir = tempfile.mkdtemp()
ebuildpath = os.path.join(tmpdir, ebuild_file_name)
file = open(ebuildpath, "w")
@@ -2986,17 +3020,25 @@ def validate_ebuild_environment(trees):
check_locale()
-def check_procfs():
- procfs_path = "/proc"
- if platform.system() not in ("Linux",) or os.path.ismount(procfs_path):
- return os.EX_OK
- msg = "It seems that %s is not mounted. You have been warned." % procfs_path
- writemsg_level(
- "".join("!!! %s\n" % l for l in textwrap.wrap(msg, 70)),
- level=logging.ERROR,
- noiselevel=-1,
- )
- return 1
+def check_mounted_fs():
+ """We need /proc for finding CPU counts and finding other system information.
+ We need /run for e.g. lock files in ebuilds."""
+ paths = {"/proc": False, "/run": False}
+
+ for path in paths.keys():
+ if platform.system() not in ("Linux",) or os.path.ismount(path):
+ paths[path] = True
+ continue
+
+ msg = "It seems %s is not mounted. Process management may malfunction." % path
+ writemsg_level(
+ "".join("!!! %s\n" % l for l in textwrap.wrap(msg, 70)),
+ level=logging.ERROR,
+ noiselevel=-1,
+ )
+
+ # Were all of the mounts we were looking for available?
+ return all(paths.values())
def config_protect_check(trees):
@@ -3474,7 +3516,8 @@ def run_action(emerge_config):
repo_name_check(emerge_config.trees)
repo_name_duplicate_check(emerge_config.trees)
config_protect_check(emerge_config.trees)
- check_procfs()
+
+ check_mounted_fs()
for mytrees in emerge_config.trees.values():
mydb = mytrees["porttree"].dbapi
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 22d0be196..14a71a610 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -15,7 +15,12 @@ from itertools import chain
import portage
from portage import os
from portage import _unicode_decode, _unicode_encode, _encodings
-from portage.const import PORTAGE_PACKAGE_ATOM, USER_CONFIG_PATH, VCS_DIRS
+from portage.const import (
+ PORTAGE_PACKAGE_ATOM,
+ USER_CONFIG_PATH,
+ VCS_DIRS,
+ SUPPORTED_GPKG_EXTENSIONS,
+)
from portage.dbapi import dbapi
from portage.dbapi.dep_expand import dep_expand
from portage.dbapi.DummyTree import DummyTree
@@ -55,6 +60,7 @@ from portage.util.digraph import digraph
from portage.util.futures import asyncio
from portage.util._async.TaskScheduler import TaskScheduler
from portage.versions import _pkg_str, catpkgsplit
+from portage.binpkg import get_binpkg_format
from _emerge.AtomArg import AtomArg
from _emerge.Blocker import Blocker
@@ -4558,8 +4564,7 @@ class depgraph:
onlydeps = "--onlydeps" in self._frozen_config.myopts
lookup_owners = []
for x in myfiles:
- ext = os.path.splitext(x)[1]
- if ext == ".tbz2":
+ if x.endswith(".tbz2") or x.endswith(SUPPORTED_GPKG_EXTENSIONS):
if not os.path.exists(x):
if os.path.exists(os.path.join(pkgsettings["PKGDIR"], "All", x)):
x = os.path.join(pkgsettings["PKGDIR"], "All", x)
@@ -4571,13 +4576,22 @@ class depgraph:
noiselevel=-1,
)
writemsg(
- "!!! Please ensure the tbz2 exists as specified.\n\n",
+ "!!! Please ensure the binpkg exists as specified.\n\n",
noiselevel=-1,
)
return 0, myfavorites
- mytbz2 = portage.xpak.tbz2(x)
- mykey = None
- cat = mytbz2.getfile("CATEGORY")
+ binpkg_format = get_binpkg_format(x)
+ if binpkg_format == "xpak":
+ mytbz2 = portage.xpak.tbz2(x)
+ mykey = None
+ cat = mytbz2.getfile("CATEGORY")
+ elif binpkg_format == "gpkg":
+ mygpkg = portage.gpkg.gpkg(self.frozen_config, None, x)
+ mykey = None
+ cat = mygpkg.get_metadata("CATEGORY")
+ else:
+ raise InvalidBinaryPackageFormat(x)
+
if cat is not None:
cat = _unicode_decode(
cat.strip(), encoding=_encodings["repo.content"]
@@ -4619,7 +4633,7 @@ class depgraph:
return 0, myfavorites
args.append(PackageArg(arg=x, package=pkg, root_config=root_config))
- elif ext == ".ebuild":
+ elif x.endswith(".ebuild"):
ebuild_path = portage.util.normalize_path(os.path.abspath(x))
pkgdir = os.path.dirname(ebuild_path)
tree_root = os.path.dirname(os.path.dirname(pkgdir))
@@ -4806,21 +4820,6 @@ class depgraph:
if len(non_virtual_cps) == 1:
installed_cp_set = non_virtual_cps
- if len(expanded_atoms) > 1 and len(installed_cp_set) == 1:
- installed_cp = next(iter(installed_cp_set))
- for atom in expanded_atoms:
- if atom.cp == installed_cp:
- available = False
- for pkg in self._iter_match_pkgs_any(
- root_config, atom.without_use, onlydeps=onlydeps
- ):
- if not pkg.installed:
- available = True
- break
- if available:
- expanded_atoms = [atom]
- break
-
# If a non-virtual package and one or more virtual packages
# are in expanded_atoms, use the non-virtual package.
if len(expanded_atoms) > 1:
@@ -7303,6 +7302,7 @@ class depgraph:
rebuilt_binaries = "rebuilt_binaries" in self._dynamic_config.myparams
usepkg = "--usepkg" in self._frozen_config.myopts
usepkgonly = "--usepkgonly" in self._frozen_config.myopts
+ usepkg_exclude_live = "--usepkg-exclude-live" in self._frozen_config.myopts
empty = "empty" in self._dynamic_config.myparams
selective = "selective" in self._dynamic_config.myparams
reinstall = False
@@ -7379,6 +7379,17 @@ class depgraph:
):
break
+ # We can choose not to install a live package from using binary
+ # cache by disabling it with option --usepkg-exclude-live in the
+ # emerge call.
+ if (
+ usepkg_exclude_live
+ and built
+ and not installed
+ and "live" in pkg._metadata.get("PROPERTIES", "").split()
+ ):
+ continue
+
useoldpkg = useoldpkg_atoms.findAtomForPackage(
pkg, modified_use=self._pkg_use_enabled(pkg)
)
diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py
index 34555b26c..8928f268d 100644
--- a/lib/_emerge/main.py
+++ b/lib/_emerge/main.py
@@ -179,6 +179,7 @@ def insert_optional_args(args):
"--use-ebuild-visibility": y_or_n,
"--usepkg": y_or_n,
"--usepkgonly": y_or_n,
+ "--usepkg-exclude-live": y_or_n,
"--verbose": y_or_n,
"--verbose-slot-rebuilds": y_or_n,
"--with-test-deps": y_or_n,
@@ -471,6 +472,7 @@ def parse_opts(tmpcmdline, silent=False):
"choices": y_or_n,
},
"--exclude": {
+ "shortopt": "-X",
"help": "A space separated list of package names or slot atoms. "
+ "Emerge won't install any ebuild or binary package that "
+ "matches any of the given package atoms.",
@@ -720,6 +722,10 @@ def parse_opts(tmpcmdline, silent=False):
"help": "use only binary packages",
"choices": true_y_or_n,
},
+ "--usepkg-exclude-live": {
+ "help": "do not install from binary packages for live ebuilds",
+ "choices": true_y_or_n,
+ },
"--verbose": {
"shortopt": "-v",
"help": "verbose output",
@@ -1115,6 +1121,11 @@ def parse_opts(tmpcmdline, silent=False):
else:
myoptions.usepkgonly = None
+ if myoptions.usepkg_exclude_live in true_y:
+ myoptions.usepkg_exclude_live = True
+ else:
+ myoptions.usepkg_exclude_live = None
+
if myoptions.verbose in true_y:
myoptions.verbose = True
else:
diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py
index 13af8da09..3042de1aa 100644
--- a/lib/portage/__init__.py
+++ b/lib/portage/__init__.py
@@ -124,6 +124,7 @@ try:
+ "cpv_getkey@getCPFromCPV,endversion_keys,"
+ "suffix_value@endversion,pkgcmp,pkgsplit,vercmp,ververify",
"portage.xpak",
+ "portage.gpkg",
"subprocess",
"time",
)
diff --git a/lib/portage/binpkg.py b/lib/portage/binpkg.py
new file mode 100644
index 000000000..ed2fda827
--- /dev/null
+++ b/lib/portage/binpkg.py
@@ -0,0 +1,56 @@
+# Copyright 2001-2020 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import tarfile
+from portage.const import SUPPORTED_XPAK_EXTENSIONS, SUPPORTED_GPKG_EXTENSIONS
+from portage.output import colorize
+from portage.util import writemsg
+
+
+def get_binpkg_format(binpkg_path):
+ if binpkg_path.endswith(SUPPORTED_XPAK_EXTENSIONS):
+ file_ext_format = "xpak"
+ elif binpkg_path.endswith(SUPPORTED_GPKG_EXTENSIONS):
+ file_ext_format = "gpkg"
+ else:
+ file_ext_format = None
+
+ try:
+ with open(binpkg_path, "rb") as binpkg_file:
+ header = binpkg_file.read(6)
+ if header == b"gpkg-1":
+ file_format = "gpkg"
+ else:
+ binpkg_file.seek(-16, 2)
+ tail = binpkg_file.read(16)
+ if (tail[0:8] == b"XPAKSTOP") and (tail[12:16] == b"STOP"):
+ file_format = "xpak"
+ else:
+ file_format = None
+
+ # check if wrong order gpkg
+ if file_format is None:
+ try:
+ with tarfile.open(binpkg_path) as gpkg_tar:
+ if "gpkg-1" in gpkg_tar.getnames():
+ file_format = "gpkg"
+ except tarfile.TarError:
+ pass
+
+ except OSError:
+ file_format = None
+
+ if file_format is None:
+ return None
+
+ if (file_ext_format is not None) and (file_ext_format != file_format):
+ writemsg(
+ colorize(
+ "WARN",
+ "File {} binpkg format mismatch, actual format: {}".format(
+ binpkg_path, file_format
+ ),
+ )
+ )
+
+ return file_format
diff --git a/lib/portage/const.py b/lib/portage/const.py
index abe0ef6c6..9e6f9311d 100644
--- a/lib/portage/const.py
+++ b/lib/portage/const.py
@@ -3,6 +3,7 @@
# Distributed under the terms of the GNU General Public License v2
import os
+import sys
# ===========================================================================
# START OF CONSTANTS -- START OF CONSTANTS -- START OF CONSTANTS -- START OF
@@ -128,9 +129,13 @@ SUPPORTED_FEATURES = frozenset(
"assume-digests",
"binpkg-docompress",
"binpkg-dostrip",
+ "binpkg-ignore-signature",
"binpkg-logs",
"binpkg-multi-instance",
+ "binpkg-request-signature",
+ "binpkg-signing",
"buildpkg",
+ "buildpkg-live",
"buildsyspkg",
"candy",
"case-insensitive-fs",
@@ -154,6 +159,7 @@ SUPPORTED_FEATURES = frozenset(
"force-mirror",
"force-prefix",
"getbinpkg",
+ "gpg-keepalive",
"icecream",
"installsources",
"ipc-sandbox",
@@ -255,7 +261,14 @@ LIVE_ECLASSES = frozenset(
)
SUPPORTED_BINPKG_FORMATS = ("tar", "rpm")
+
+if sys.version_info.major < 3:
+ SUPPORTED_GENTOO_BINPKG_FORMATS = ("xpak",)
+else:
+ SUPPORTED_GENTOO_BINPKG_FORMATS = ("xpak", "gpkg")
+
SUPPORTED_XPAK_EXTENSIONS = (".tbz2", ".xpak")
+SUPPORTED_GPKG_EXTENSIONS = (".gpkg.tar",)
# Time formats used in various places like metadata.chk.
TIMESTAMP_FORMAT = "%a, %d %b %Y %H:%M:%S +0000" # to be used with time.gmtime()
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index 777fc4918..8bfe5e97d 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -26,20 +26,29 @@ portage.proxy.lazyimport.lazyimport(
from portage.binrepo.config import BinRepoConfigLoader
from portage.cache.mappings import slot_dict_class
-from portage.const import BINREPOS_CONF_FILE, CACHE_PATH, SUPPORTED_XPAK_EXTENSIONS
+from portage.const import (
+ BINREPOS_CONF_FILE,
+ CACHE_PATH,
+ SUPPORTED_XPAK_EXTENSIONS,
+ SUPPORTED_GPKG_EXTENSIONS,
+ SUPPORTED_GENTOO_BINPKG_FORMATS,
+)
from portage.dbapi.virtual import fakedbapi
from portage.dep import Atom, use_reduce, paren_enclose
from portage.exception import (
AlarmSignal,
InvalidPackageName,
+ InvalidBinaryPackageFormat,
ParseError,
PortageException,
+ PortagePackageException,
)
from portage.localization import _
from portage.package.ebuild.profile_iuse import iter_iuse_vars
from portage.util.file_copy import copyfile
from portage.util.futures import asyncio
from portage.util.futures.executor.fork import ForkExecutor
+from portage.binpkg import get_binpkg_format
from portage import _movefile
from portage import os
from portage import _encodings
@@ -49,6 +58,7 @@ from portage import _unicode_encode
import codecs
import errno
import io
+import re
import stat
import subprocess
import tempfile
@@ -72,6 +82,7 @@ class bindbapi(fakedbapi):
list(fakedbapi._known_keys) + ["CHOST", "repository", "USE"]
)
_pkg_str_aux_keys = fakedbapi._pkg_str_aux_keys + (
+ "BINPKG_FORMAT",
"BUILD_ID",
"BUILD_TIME",
"_mtime_",
@@ -94,6 +105,7 @@ class bindbapi(fakedbapi):
self._aux_cache_keys = set(
[
"BDEPEND",
+ "BINPKG_FORMAT",
"BUILD_ID",
"BUILD_TIME",
"CHOST",
@@ -170,28 +182,44 @@ class bindbapi(fakedbapi):
return add_pkg._db.aux_get(add_pkg, wants)
if not self.bintree._remotepkgs or not self.bintree.isremote(mycpv):
try:
- tbz2_path = self.bintree._pkg_paths[instance_key]
+ binpkg_path = self.bintree._pkg_paths[instance_key]
except KeyError:
raise KeyError(mycpv)
- tbz2_path = os.path.join(self.bintree.pkgdir, tbz2_path)
+ binpkg_path = os.path.join(self.bintree.pkgdir, binpkg_path)
try:
- st = os.lstat(tbz2_path)
+ st = os.lstat(binpkg_path)
except OSError:
raise KeyError(mycpv)
- metadata_bytes = portage.xpak.tbz2(tbz2_path).get_data()
+ binpkg_format = self.cpvdict[instance_key]["BINPKG_FORMAT"]
+ if binpkg_format == "xpak":
+ metadata_bytes = portage.xpak.tbz2(binpkg_path).get_data()
+ decode_metadata_name = False
+ elif binpkg_format == "gpkg":
+ metadata_bytes = portage.gpkg.gpkg(
+ self.settings, mycpv, binpkg_path
+ ).get_metadata()
+ decode_metadata_name = True
+ else:
+ raise InvalidBinaryPackageFormat(
+ "Unknown binary package format %s" % binpkg_path
+ )
def getitem(k):
if k == "_mtime_":
return str(st[stat.ST_MTIME])
if k == "SIZE":
return str(st.st_size)
- v = metadata_bytes.get(
- _unicode_encode(
- k,
- encoding=_encodings["repo.content"],
- errors="backslashreplace",
- )
- )
+ else:
+ if decode_metadata_name:
+ v = metadata_bytes.get(k)
+ else:
+ v = metadata_bytes.get(
+ _unicode_encode(
+ k,
+ encoding=_encodings["repo.content"],
+ errors="backslashreplace",
+ )
+ )
if v is not None:
v = _unicode_decode(
v, encoding=_encodings["repo.content"], errors="replace"
@@ -202,6 +230,7 @@ class bindbapi(fakedbapi):
getitem = self.cpvdict[instance_key].get
mydata = {}
mykeys = wants
+
for x in mykeys:
myval = getitem(x)
# myval is None if the key doesn't exist
@@ -230,16 +259,29 @@ class bindbapi(fakedbapi):
cpv = self._instance_key(cpv, support_string=True)[0]
build_id = cpv.build_id
- tbz2path = self.bintree.getname(cpv)
- if not os.path.exists(tbz2path):
+ binpkg_path = self.bintree.getname(cpv)
+ if not os.path.exists(binpkg_path):
raise KeyError(cpv)
- mytbz2 = portage.xpak.tbz2(tbz2path)
- mydata = mytbz2.get_data()
- for k, v in values.items():
- k = _unicode_encode(
- k, encoding=_encodings["repo.content"], errors="backslashreplace"
+ binpkg_format = cpv.binpkg_format
+ if binpkg_format == "xpak":
+ mytbz2 = portage.xpak.tbz2(binpkg_path)
+ mydata = mytbz2.get_data()
+ encoding_key = True
+ elif binpkg_format == "gpkg":
+ mybinpkg = portage.gpkg.gpkg(self.settings, cpv, binpkg_path)
+ mydata = mybinpkg.get_metadata()
+ encoding_key = False
+ else:
+ raise InvalidBinaryPackageFormat(
+ "Unknown binary package format %s" % binpkg_path
)
+
+ for k, v in values.items():
+ if encoding_key:
+ k = _unicode_encode(
+ k, encoding=_encodings["repo.content"], errors="backslashreplace"
+ )
v = _unicode_encode(
v, encoding=_encodings["repo.content"], errors="backslashreplace"
)
@@ -248,7 +290,15 @@ class bindbapi(fakedbapi):
for k, v in list(mydata.items()):
if not v:
del mydata[k]
- mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
+ if binpkg_format == "xpak":
+ mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
+ elif binpkg_format == "gpkg":
+ mybinpkg.update_metadata(mydata)
+ else:
+ raise InvalidBinaryPackageFormat(
+ "Unknown binary package format %s" % binpkg_path
+ )
+
# inject will clear stale caches via cpv_inject.
self.bintree.inject(cpv)
@@ -271,12 +321,24 @@ class bindbapi(fakedbapi):
if add_pkg is not None:
await add_pkg._db.unpack_metadata(pkg, dest_dir, loop=loop)
else:
- tbz2_file = self.bintree.getname(cpv)
- await loop.run_in_executor(
- ForkExecutor(loop=loop),
- portage.xpak.tbz2(tbz2_file).unpackinfo,
- dest_dir,
- )
+ binpkg_file = self.bintree.getname(cpv)
+ binpkg_format = cpv.binpkg_format
+ if binpkg_format == "xpak":
+ await loop.run_in_executor(
+ ForkExecutor(loop=loop),
+ portage.xpak.tbz2(binpkg_file).unpackinfo,
+ dest_dir,
+ )
+ elif binpkg_format == "gpkg":
+ await loop.run_in_executor(
+ ForkExecutor(loop=loop),
+ portage.gpkg.gpkg(self.settings, cpv, binpkg_file).unpack_metadata,
+ dest_dir,
+ )
+ else:
+ raise InvalidBinaryPackageFormat(
+ "Unknown binary package format %s" % binpkg_file
+ )
async def unpack_contents(self, pkg, dest_dir, loop=None):
"""
@@ -297,23 +359,31 @@ class bindbapi(fakedbapi):
pkg_path = self.bintree.getname(cpv)
if pkg_path is not None:
+ binpkg_format = cpv.binpkg_format
+ if binpkg_format == "xpak":
+ extractor = BinpkgExtractorAsync(
+ background=settings.get("PORTAGE_BACKGROUND") == "1",
+ env=settings.environ(),
+ features=settings.features,
+ image_dir=dest_dir,
+ pkg=cpv,
+ pkg_path=pkg_path,
+ logfile=settings.get("PORTAGE_LOG_FILE"),
+ scheduler=SchedulerInterface(loop),
+ )
- extractor = BinpkgExtractorAsync(
- background=settings.get("PORTAGE_BACKGROUND") == "1",
- env=settings.environ(),
- features=settings.features,
- image_dir=dest_dir,
- pkg=cpv,
- pkg_path=pkg_path,
- logfile=settings.get("PORTAGE_LOG_FILE"),
- scheduler=SchedulerInterface(loop),
- )
-
- extractor.start()
- await extractor.async_wait()
- if extractor.returncode != os.EX_OK:
- raise PortageException("Error Extracting '{}'".format(pkg_path))
-
+ extractor.start()
+ await extractor.async_wait()
+ if extractor.returncode != os.EX_OK:
+ raise PortageException("Error Extracting '{}'".format(pkg_path))
+ elif binpkg_format == "gpkg":
+ await loop.run_in_executor(
+ ForkExecutor(loop=loop),
+ portage.gpkg.gpkg(self.settings, cpv, pkg_path).decompress,
+ dest_dir,
+ )
+ else:
+ raise portage.exception.InvalidBinaryPackageFormat(pkg_path)
else:
instance_key = self._instance_key(cpv)
add_pkg = self.bintree._additional_pkgs.get(instance_key)
@@ -430,6 +500,7 @@ class binarytree:
self._pkgindex_aux_keys = [
"BASE_URI",
"BDEPEND",
+ "BINPKG_FORMAT",
"BUILD_ID",
"BUILD_TIME",
"CHOST",
@@ -473,6 +544,7 @@ class binarytree:
"ACCEPT_LICENSE",
"ACCEPT_PROPERTIES",
"ACCEPT_RESTRICT",
+ "BINPKG_FORMAT",
"CBUILD",
"CONFIG_PROTECT",
"CONFIG_PROTECT_MASK",
@@ -508,10 +580,13 @@ class binarytree:
"SLOT": "0",
"USE": "",
}
- self._pkgindex_inherited_keys = ["CHOST", "repository"]
+ self._pkgindex_inherited_keys = ["BINPKG_FORMAT", "CHOST", "repository"]
# Populate the header with appropriate defaults.
self._pkgindex_default_header_data = {
+ "BINPKG_FORMAT": self.settings.get(
+ "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
+ ),
"CHOST": self.settings.get("CHOST", ""),
"repository": "",
}
@@ -594,24 +669,42 @@ class binarytree:
writemsg("!!! " + mycpv + " -> " + mynewcpv + "\n", noiselevel=-1)
continue
- tbz2path = self.getname(mycpv)
- if os.path.exists(tbz2path) and not os.access(tbz2path, os.W_OK):
+ binpkg_path = self.getname(mycpv)
+ if os.path.exists(binpkg_path) and not os.access(binpkg_path, os.W_OK):
writemsg(
_("!!! Cannot update readonly binary: %s\n") % mycpv, noiselevel=-1
)
continue
moves += 1
- mytbz2 = portage.xpak.tbz2(tbz2path)
- mydata = mytbz2.get_data()
+ binpkg_format = mycpv.binpkg_format
+ if binpkg_format == "xpak":
+ mytbz2 = portage.xpak.tbz2(binpkg_path)
+ mydata = mytbz2.get_data()
+ decode_metadata_name = False
+ elif binpkg_format == "gpkg":
+ mybinpkg = portage.gpkg.gpkg(self.settings, mycpv, binpkg_path)
+ mydata = mybinpkg.get_metadata()
+ decode_metadata_name = True
+ else:
+ continue
+
updated_items = update_dbentries([mylist], mydata, parent=mycpv)
mydata.update(updated_items)
- mydata[b"PF"] = _unicode_encode(
- mynewpkg + "\n", encoding=_encodings["repo.content"]
- )
- mydata[b"CATEGORY"] = _unicode_encode(
- mynewcat + "\n", encoding=_encodings["repo.content"]
- )
+ if decode_metadata_name:
+ mydata["PF"] = _unicode_encode(
+ mynewpkg + "\n", encoding=_encodings["repo.content"]
+ )
+ mydata["CATEGORY"] = _unicode_encode(
+ mynewcat + "\n", encoding=_encodings["repo.content"]
+ )
+ else:
+ mydata[b"PF"] = _unicode_encode(
+ mynewpkg + "\n", encoding=_encodings["repo.content"]
+ )
+ mydata[b"CATEGORY"] = _unicode_encode(
+ mynewcat + "\n", encoding=_encodings["repo.content"]
+ )
if mynewpkg != myoldpkg:
ebuild_data = mydata.pop(
_unicode_encode(
@@ -628,7 +721,7 @@ class binarytree:
metadata = self.dbapi._aux_cache_slot_dict()
for k in self.dbapi._aux_cache_keys:
- v = mydata.get(_unicode_encode(k))
+ v = mydata.get(k if decode_metadata_name else _unicode_encode(k))
if v is not None:
v = _unicode_decode(v)
metadata[k] = " ".join(v.split())
@@ -643,9 +736,15 @@ class binarytree:
update_path_lock = None
try:
update_path_lock = lockfile(update_path, wantnewlockfile=True)
- copyfile(tbz2path, update_path)
- mytbz2 = portage.xpak.tbz2(update_path)
- mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
+ copyfile(binpkg_path, update_path)
+ if binpkg_format == "xpak":
+ mytbz2 = portage.xpak.tbz2(update_path)
+ mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
+ elif binpkg_format == "gpkg":
+ mybinpkg = portage.gpkg.gpkg(self.settings, mycpv, update_path)
+ mybinpkg.update_metadata(mydata, newcpv=mynewcpv)
+ else:
+ raise InvalidBinaryPackageFormat(binpkg_format)
self.inject(mynewcpv, filename=update_path)
finally:
if update_path_lock is not None:
@@ -810,7 +909,13 @@ class binarytree:
metadata[_instance_key(cpv)] = d
path = d.get("PATH")
if not path:
- path = cpv + ".tbz2"
+ binpkg_format = d["BINPKG_FORMAT"]
+ if binpkg_format == "xpak":
+ path = cpv + ".tbz2"
+ elif binpkg_format == "gpkg":
+ path = cpv + ".gpkg.tar"
+ else:
+ continue
if reindex:
basename = os.path.basename(path)
@@ -835,7 +940,9 @@ class binarytree:
)
except UnicodeDecodeError:
continue
- if not myfile.endswith(SUPPORTED_XPAK_EXTENSIONS):
+ if not myfile.endswith(
+ SUPPORTED_XPAK_EXTENSIONS + SUPPORTED_GPKG_EXTENSIONS
+ ):
continue
mypath = os.path.join(mydir, myfile)
full_path = os.path.join(self.pkgdir, mypath)
@@ -846,9 +953,9 @@ class binarytree:
# Validate data from the package index and try to avoid
# reading the xpak if possible.
+ match = None
possibilities = basename_index.get(myfile)
if possibilities:
- match = None
for d in possibilities:
try:
if int(d["_mtime_"]) != s[stat.ST_MTIME]:
@@ -873,7 +980,9 @@ class binarytree:
update_pkgindex = True
# Omit PATH if it is the default path for
# the current Packages format version.
- if mypath != mycpv + ".tbz2":
+ if (mypath != mycpv + ".tbz2") and (
+ mypath != mycpv + ".gpkg.tar"
+ ):
d["PATH"] = mypath
if not oldpath:
update_pkgindex = True
@@ -891,15 +1000,30 @@ class binarytree:
)
self.invalids.append(myfile[:-5])
continue
- pkg_metadata = self._read_metadata(
- full_path,
- s,
- keys=chain(self.dbapi._aux_cache_keys, ("PF", "CATEGORY")),
- )
+
+ binpkg_format = None
+ if match:
+ binpkg_format = match.get("BINPKG_FORMAT", None)
+ try:
+ pkg_metadata = self._read_metadata(
+ full_path,
+ s,
+ keys=chain(self.dbapi._aux_cache_keys, ("PF", "CATEGORY")),
+ binpkg_format=binpkg_format,
+ )
+ except PortagePackageException as e:
+ writemsg(
+ f"!!! Invalid binary package: '{full_path}', {e}\n",
+ noiselevel=-1,
+ )
+ continue
mycat = pkg_metadata.get("CATEGORY", "")
mypf = pkg_metadata.get("PF", "")
slot = pkg_metadata.get("SLOT", "")
- mypkg = myfile[:-5]
+ for ext in SUPPORTED_XPAK_EXTENSIONS + SUPPORTED_GPKG_EXTENSIONS:
+ if myfile.endswith(ext):
+ mypkg = myfile[: -len(ext)]
+ break
if not mycat or not mypf or not slot:
# old-style or corrupt package
writemsg(
@@ -943,9 +1067,19 @@ class binarytree:
invalid_name = True
else:
mypkg = mypkg[: -len(str(build_id)) - 1]
+ elif myfile.endswith(".gpkg.tar"):
+ build_id = self._parse_build_id(myfile)
+ if build_id > 0:
+ multi_instance = True
+ if myfile != "%s-%s.gpkg.tar" % (mypf, build_id):
+ invalid_name = True
+ else:
+ mypkg = mypkg[: -len(str(build_id)) - 1]
+ else:
+ if myfile != "%s.gpkg.tar" % mypf:
+ invalid_name = True
elif myfile != mypf + ".tbz2":
invalid_name = True
-
if invalid_name:
writemsg(
_("\n!!! Binary package name is " "invalid: '%s'\n")
@@ -1043,7 +1177,7 @@ class binarytree:
del pkg_paths[_instance_key(mycpv)]
# record location if it's non-default
- if mypath != mycpv + ".tbz2":
+ if (mypath != mycpv + ".tbz2") and (mypath != mycpv + ".gpkg.tar"):
d["PATH"] = mypath
else:
d.pop("PATH", None)
@@ -1407,7 +1541,10 @@ class binarytree:
noiselevel=-1,
)
return
+
metadata = self._read_metadata(full_path, s)
+ binpkg_format = metadata["BINPKG_FORMAT"]
+
invalid_depend = False
try:
self._eval_use_flags(cpv, metadata)
@@ -1455,21 +1592,35 @@ class binarytree:
basename = os.path.basename(full_path)
pf = catsplit(cpv)[1]
- if build_id is None and not fetched and basename.endswith(".xpak"):
+ if (build_id is None) and (not fetched) and binpkg_format:
# Apply the newly assigned BUILD_ID. This is intended
# to occur only for locally built packages. If the
# package was fetched, we want to preserve its
# attributes, so that we can later distinguish that it
# is identical to its remote counterpart.
build_id = self._parse_build_id(basename)
- metadata["BUILD_ID"] = str(build_id)
- cpv = _pkg_str(
- cpv, metadata=metadata, settings=self.settings, db=self.dbapi
- )
- binpkg = portage.xpak.tbz2(full_path)
- binary_data = binpkg.get_data()
- binary_data[b"BUILD_ID"] = _unicode_encode(metadata["BUILD_ID"])
- binpkg.recompose_mem(portage.xpak.xpak_mem(binary_data))
+ if build_id > 0:
+ metadata["BUILD_ID"] = str(build_id)
+ cpv = _pkg_str(
+ cpv, metadata=metadata, settings=self.settings, db=self.dbapi
+ )
+ if binpkg_format == "xpak":
+ if basename.endswith(".xpak"):
+ binpkg = portage.xpak.tbz2(full_path)
+ binary_data = binpkg.get_data()
+ binary_data[b"BUILD_ID"] = _unicode_encode(
+ metadata["BUILD_ID"]
+ )
+ binpkg.recompose_mem(portage.xpak.xpak_mem(binary_data))
+ elif binpkg_format == "gpkg":
+ binpkg = portage.gpkg.gpkg(self.settings, cpv, full_path)
+ binpkg_metadata = binpkg.get_metadata()
+ binpkg_metadata["BUILD_ID"] = _unicode_encode(
+ metadata["BUILD_ID"]
+ )
+ binpkg.update_metadata(binpkg_metadata)
+ else:
+ raise InvalidBinaryPackageFormat(basename)
self._file_permissions(full_path)
pkgindex = self._load_pkgindex()
@@ -1490,7 +1641,7 @@ class binarytree:
return cpv
- def _read_metadata(self, filename, st, keys=None):
+ def _read_metadata(self, filename, st, keys=None, binpkg_format=None):
"""
Read metadata from a binary package. The returned metadata
dictionary will contain empty strings for any values that
@@ -1511,14 +1662,35 @@ class binarytree:
metadata = self.dbapi._aux_cache_slot_dict()
else:
metadata = {}
- binary_metadata = portage.xpak.tbz2(filename).get_data()
+
+ # xpak return key as binary, gpkg return key as str
+ decode_metadata_name = True
+
+ if not binpkg_format:
+ binpkg_format = get_binpkg_format(filename)
+ if binpkg_format == "xpak":
+ binpkg_metadata = portage.xpak.tbz2(filename).get_data()
+ elif binpkg_format == "gpkg":
+ binpkg_metadata = portage.gpkg.gpkg(
+ self.settings, None, filename
+ ).get_metadata()
+ decode_metadata_name = False
+ else:
+ raise InvalidBinaryPackageFormat(
+ f"Unrecognized binary package format in '{filename}'"
+ )
+
for k in keys:
if k == "_mtime_":
metadata[k] = str(st[stat.ST_MTIME])
elif k == "SIZE":
metadata[k] = str(st.st_size)
else:
- v = binary_metadata.get(_unicode_encode(k))
+ if decode_metadata_name:
+ v = binpkg_metadata.get(_unicode_encode(k))
+ else:
+ # check gpkg
+ v = binpkg_metadata.get(k)
if v is None:
if k == "EAPI":
metadata[k] = "0"
@@ -1527,6 +1699,9 @@ class binarytree:
else:
v = _unicode_decode(v)
metadata[k] = " ".join(v.split())
+
+ metadata["BINPKG_FORMAT"] = binpkg_format
+
return metadata
def _inject_file(self, pkgindex, cpv, filename):
@@ -1608,6 +1783,10 @@ class binarytree:
"""
pkg_path = self.getname(cpv)
+ try:
+ binpkg_format = cpv.binpkg_format
+ except AttributeError:
+ raise KeyError("{} metadata not found!".format(cpv))
d = dict(cpv._metadata.items())
d.update(perform_multiple_checksums(pkg_path, hashes=self._pkgindex_hashes))
@@ -1616,11 +1795,18 @@ class binarytree:
st = os.lstat(pkg_path)
d["_mtime_"] = str(st[stat.ST_MTIME])
d["SIZE"] = str(st.st_size)
+ d["BINPKG_FORMAT"] = binpkg_format
rel_path = pkg_path[len(self.pkgdir) + 1 :]
# record location if it's non-default
- if rel_path != cpv + ".tbz2":
- d["PATH"] = rel_path
+ if binpkg_format == "xpak":
+ if rel_path != cpv + ".tbz2":
+ d["PATH"] = rel_path
+ elif binpkg_format == "gpkg":
+ if rel_path != cpv + ".gpkg.tar":
+ d["PATH"] = rel_path
+ else:
+ raise InvalidBinaryPackageFormat(binpkg_format)
return d
@@ -1822,11 +2008,45 @@ class binarytree:
return None
if filename is None:
- if self._multi_instance:
- pf = catsplit(cpv)[1]
- filename = "%s-%s.xpak" % (os.path.join(self.pkgdir, cpv.cp, pf), "1")
+ try:
+ binpkg_format = cpv.binpkg_format
+ except AttributeError:
+ # In order to force the caller to clarify its intent, do not
+ # use default BINPKG_FORMAT unless allocate_new is True.
+ # The caller can set cpv.binpkg_format in advance if something
+ # other than the default is desired here.
+ if allocate_new:
+ binpkg_format = self.settings.get(
+ "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
+ )
+ else:
+ binpkg_format = None
+
+ if not binpkg_format:
+ # Raise an error if the desired binpkg_format is not clear.
+ # The caller should either set allocate_new to True or else
+ # ensure that cpv.binpkg_format is set to a particular format.
+ raise InvalidBinaryPackageFormat(binpkg_format)
+ elif binpkg_format == "xpak":
+ if self._multi_instance:
+ pf = catsplit(cpv)[1]
+ filename = "%s-%s.xpak" % (
+ os.path.join(self.pkgdir, cpv.cp, pf),
+ "1",
+ )
+ else:
+ filename = os.path.join(self.pkgdir, cpv + ".tbz2")
+ elif binpkg_format == "gpkg":
+ if self._multi_instance:
+ pf = catsplit(cpv)[1]
+ filename = "%s-%s.gpkg.tar" % (
+ os.path.join(self.pkgdir, cpv.cp, pf),
+ "1",
+ )
+ else:
+ filename = os.path.join(self.pkgdir, cpv + ".gpkg.tar")
else:
- filename = os.path.join(self.pkgdir, cpv + ".tbz2")
+ raise InvalidBinaryPackageFormat(binpkg_format)
return filename
@@ -1850,7 +2070,19 @@ class binarytree:
return max_build_id
def _allocate_filename(self, cpv):
- return os.path.join(self.pkgdir, cpv + ".tbz2")
+ try:
+ binpkg_format = cpv.binpkg_format
+ except AttributeError:
+ binpkg_format = self.settings.get(
+ "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
+ )
+
+ if binpkg_format == "xpak":
+ return os.path.join(self.pkgdir, cpv + ".tbz2")
+ elif binpkg_format == "gpkg":
+ return os.path.join(self.pkgdir, cpv + ".gpkg.tar")
+ else:
+ raise InvalidBinaryPackageFormat(binpkg_format)
def _allocate_filename_multi(self, cpv):
@@ -1864,8 +2096,25 @@ class binarytree:
pf = catsplit(cpv)[1]
build_id = max_build_id + 1
+ try:
+ binpkg_format = cpv.binpkg_format
+ except AttributeError:
+ binpkg_format = self.settings.get(
+ "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
+ )
+
+ if binpkg_format == "xpak":
+ filename_format = "%s-%s.xpak"
+ elif binpkg_format == "gpkg":
+ filename_format = "%s-%s.gpkg.tar"
+ else:
+ raise InvalidBinaryPackageFormat(binpkg_format)
+
while True:
- filename = "%s-%s.xpak" % (os.path.join(self.pkgdir, cpv.cp, pf), build_id)
+ filename = filename_format % (
+ os.path.join(self.pkgdir, cpv.cp, pf),
+ build_id,
+ )
if os.path.exists(filename):
build_id += 1
else:
@@ -1874,14 +2123,17 @@ class binarytree:
@staticmethod
def _parse_build_id(filename):
build_id = -1
- suffixlen = len(".xpak")
- hyphen = filename.rfind("-", 0, -(suffixlen + 1))
- if hyphen != -1:
- build_id = filename[hyphen + 1 : -suffixlen]
- try:
- build_id = int(build_id)
- except ValueError:
- pass
+ if filename.endswith(SUPPORTED_XPAK_EXTENSIONS):
+ suffixlen = len(".xpak")
+ elif filename.endswith(SUPPORTED_GPKG_EXTENSIONS):
+ suffixlen = len(".gpkg.tar")
+ else:
+ raise InvalidBinaryPackageFormat(filename)
+
+ filename = filename[:-suffixlen]
+ if re.match(r".*-[\w.]*\d+[\w.]*-\d+$", filename):
+ build_id = int(filename.split("-")[-1])
+
return build_id
def isremote(self, pkgname):
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 8ffb23b1c..863efe9cc 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -43,6 +43,7 @@ portage.proxy.lazyimport.lazyimport(
"portage.util._eventloop.global_event_loop:global_event_loop",
"portage.versions:best,catpkgsplit,catsplit,cpv_getkey,vercmp,"
+ "_get_slot_re,_pkgsplit@pkgsplit,_pkg_str,_unknown_repo",
+ "portage.gpkg",
"subprocess",
"tarfile",
)
@@ -54,6 +55,7 @@ from portage.const import (
PORTAGE_PACKAGE_ATOM,
PRIVATE_PATH,
VDB_PATH,
+ SUPPORTED_GENTOO_BINPKG_FORMATS,
)
from portage.dbapi import dbapi
from portage.exception import (
@@ -61,6 +63,7 @@ from portage.exception import (
InvalidData,
InvalidLocation,
InvalidPackageName,
+ InvalidBinaryPackageFormat,
FileNotFound,
PermissionDenied,
UnsupportedAPIException,
@@ -1089,26 +1092,45 @@ class vardbapi(dbapi):
"y" if include_unmodified_config else "n"
)
)
-
opts, args = parser.parse_known_args(opts_list)
- tar_cmd = ("tar", "-x", "--xattrs", "--xattrs-include=*", "-C", dest_dir)
- pr, pw = os.pipe()
- proc = await asyncio.create_subprocess_exec(*tar_cmd, stdin=pr)
- os.close(pr)
- with os.fdopen(pw, "wb", 0) as pw_file:
+ binpkg_format = settings.get(
+ "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
+ )
+ if binpkg_format == "xpak":
+ tar_cmd = ("tar", "-x", "--xattrs", "--xattrs-include=*", "-C", dest_dir)
+ pr, pw = os.pipe()
+ proc = await asyncio.create_subprocess_exec(*tar_cmd, stdin=pr)
+ os.close(pr)
+ with os.fdopen(pw, "wb", 0) as pw_file:
+ excluded_config_files = await loop.run_in_executor(
+ ForkExecutor(loop=loop),
+ functools.partial(
+ self._dblink(cpv).quickpkg,
+ pw_file,
+ include_config=opts.include_config == "y",
+ include_unmodified_config=opts.include_unmodified_config == "y",
+ ),
+ )
+ await proc.wait()
+ if proc.returncode != os.EX_OK:
+ raise PortageException("command failed: {}".format(tar_cmd))
+ elif binpkg_format == "gpkg":
+ gpkg_tmp_fd, gpkg_tmp = tempfile.mkstemp(suffix=".gpkg.tar")
+ os.close(gpkg_tmp_fd)
excluded_config_files = await loop.run_in_executor(
ForkExecutor(loop=loop),
functools.partial(
self._dblink(cpv).quickpkg,
- pw_file,
+ gpkg_tmp,
include_config=opts.include_config == "y",
include_unmodified_config=opts.include_unmodified_config == "y",
),
)
- await proc.wait()
- if proc.returncode != os.EX_OK:
- raise PortageException("command failed: {}".format(tar_cmd))
+ portage.gpkg.gpkg(settings, cpv, gpkg_tmp).decompress(dest_dir)
+ os.remove(gpkg_tmp)
+ else:
+ raise InvalidBinaryPackageFormat(binpkg_format)
if excluded_config_files:
log_lines = [
@@ -2107,7 +2129,11 @@ class dblink:
return pkgfiles
def quickpkg(
- self, output_file, include_config=False, include_unmodified_config=False
+ self,
+ output_file,
+ metadata=None,
+ include_config=False,
+ include_unmodified_config=False,
):
"""
Create a tar file appropriate for use by quickpkg.
@@ -2130,6 +2156,9 @@ class dblink:
contents = self.getcontents()
excluded_config_files = []
protect = None
+ binpkg_format = settings.get(
+ "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
+ )
if not include_config:
confprot = ConfigProtect(
@@ -2152,16 +2181,22 @@ class dblink:
excluded_config_files.append(filename)
return True
- # The tarfile module will write pax headers holding the
- # xattrs only if PAX_FORMAT is specified here.
- with tarfile.open(
- fileobj=output_file,
- mode="w|",
- format=tarfile.PAX_FORMAT if xattrs else tarfile.DEFAULT_FORMAT,
- ) as tar:
- tar_contents(
- contents, settings["ROOT"], tar, protect=protect, xattrs=xattrs
- )
+ if binpkg_format == "xpak":
+ # The tarfile module will write pax headers holding the
+ # xattrs only if PAX_FORMAT is specified here.
+ with tarfile.open(
+ fileobj=output_file,
+ mode="w|",
+ format=tarfile.PAX_FORMAT if xattrs else tarfile.DEFAULT_FORMAT,
+ ) as tar:
+ tar_contents(
+ contents, settings["ROOT"], tar, protect=protect, xattrs=xattrs
+ )
+ elif binpkg_format == "gpkg":
+ gpkg_file = portage.gpkg.gpkg(settings, cpv, output_file)
+ gpkg_file._quickpkg(contents, metadata, settings["ROOT"], protect=protect)
+ else:
+ raise InvalidBinaryPackageFormat(binpkg_format)
return excluded_config_files
diff --git a/lib/portage/dep/dep_check.py b/lib/portage/dep/dep_check.py
index 9fccda08b..8ca4c0b9d 100644
--- a/lib/portage/dep/dep_check.py
+++ b/lib/portage/dep/dep_check.py
@@ -376,6 +376,7 @@ def dep_zapdeps(
# c) contains masked installed packages
# d) is the first item
+ no_new_slots = []
preferred_in_graph = []
preferred_installed = preferred_in_graph
preferred_any_slot = preferred_in_graph
@@ -391,6 +392,7 @@ def dep_zapdeps(
# unsat_use_* must come after preferred_non_installed
# for correct ordering in cases like || ( foo[a] foo[b] ).
choice_bins = (
+ no_new_slots,
preferred_in_graph,
preferred_non_installed,
unsat_use_in_graph,
@@ -689,7 +691,9 @@ def dep_zapdeps(
other.append(this_choice)
else:
if all_use_satisfied:
- if all_in_graph:
+ if new_slot_count == 0 and not want_update:
+ no_new_slots.append(this_choice)
+ elif all_in_graph:
preferred_in_graph.append(this_choice)
elif all_installed:
if all_installed_slots:
diff --git a/lib/portage/dep/soname/multilib_category.py b/lib/portage/dep/soname/multilib_category.py
index ab855d339..567b1d32e 100644
--- a/lib/portage/dep/soname/multilib_category.py
+++ b/lib/portage/dep/soname/multilib_category.py
@@ -11,6 +11,7 @@
# arm_{32,64}
# hppa_{32,64}
# ia_{32,64}
+# loong_{ilp32s,ilp32f,ilp32d,lp64s,lp64f,lp64d}
# m68k_{32,64}
# mips_{eabi32,eabi64,n32,n64,o32,o64}
# ppc_{32,64}
@@ -37,6 +38,13 @@ from portage.util.elf.constants import (
EF_MIPS_ABI2,
EF_RISCV_FLOAT_ABI_DOUBLE,
EF_RISCV_RVC,
+ EF_LOONGARCH_ABI_LP64_SOFT_FLOAT,
+ EF_LOONGARCH_ABI_LP64_SINGLE_FLOAT,
+ EF_LOONGARCH_ABI_LP64_DOUBLE_FLOAT,
+ EF_LOONGARCH_ABI_ILP32_SOFT_FLOAT,
+ EF_LOONGARCH_ABI_ILP32_SINGLE_FLOAT,
+ EF_LOONGARCH_ABI_ILP32_DOUBLE_FLOAT,
+ EF_LOONGARCH_ABI_MASK,
ELFCLASS32,
ELFCLASS64,
EM_386,
@@ -46,6 +54,7 @@ from portage.util.elf.constants import (
EM_ARM,
EM_ALTERA_NIOS2,
EM_IA_64,
+ EM_LOONGARCH,
EM_MIPS,
EM_PARISC,
EM_PPC,
@@ -71,6 +80,7 @@ _machine_prefix_map = {
EM_ALTERA_NIOS2: "nios2",
EM_ARM: "arm",
EM_IA_64: "ia64",
+ EM_LOONGARCH: "loong",
EM_MIPS: "mips",
EM_PARISC: "hppa",
EM_PPC: "ppc",
@@ -84,6 +94,15 @@ _machine_prefix_map = {
EM_X86_64: "x86",
}
+_loong_abi_map = {
+ EF_LOONGARCH_ABI_LP64_SOFT_FLOAT: "lp64s",
+ EF_LOONGARCH_ABI_LP64_SINGLE_FLOAT: "lp64f",
+ EF_LOONGARCH_ABI_LP64_DOUBLE_FLOAT: "lp64d",
+ EF_LOONGARCH_ABI_ILP32_SOFT_FLOAT: "ilp32s",
+ EF_LOONGARCH_ABI_ILP32_SINGLE_FLOAT: "ilp32f",
+ EF_LOONGARCH_ABI_ILP32_DOUBLE_FLOAT: "ilp32d",
+}
+
_mips_abi_map = {
E_MIPS_ABI_EABI32: "eabi32",
E_MIPS_ABI_EABI64: "eabi64",
@@ -92,6 +111,12 @@ _mips_abi_map = {
}
+def _compute_suffix_loong(elf_header):
+
+ loong_abi = elf_header.e_flags & EF_LOONGARCH_ABI_MASK
+ return _loong_abi_map.get(loong_abi)
+
+
def _compute_suffix_mips(elf_header):
name = None
@@ -136,6 +161,7 @@ def _compute_suffix_riscv(elf_header):
_specialized_funcs = {
+ "loong": _compute_suffix_loong,
"mips": _compute_suffix_mips,
"riscv": _compute_suffix_riscv,
}
diff --git a/lib/portage/exception.py b/lib/portage/exception.py
index ec8ea1980..3df4ce8fd 100644
--- a/lib/portage/exception.py
+++ b/lib/portage/exception.py
@@ -181,6 +181,22 @@ class InvalidPackageName(PortagePackageException):
"""Malformed package name"""
+class InvalidBinaryPackageFormat(PortagePackageException):
+ """Invalid Binary Package Format"""
+
+
+class InvalidCompressionMethod(PortagePackageException):
+ """Invalid or unsupported compression method"""
+
+
+class CompressorNotFound(PortagePackageException):
+ """A required compressor binary was not available or executable"""
+
+
+class CompressorOperationFailed(PortagePackageException):
+ """An error occurred during external operation"""
+
+
class InvalidAtom(PortagePackageException):
"""Malformed atom spec"""
@@ -208,6 +224,10 @@ class UnsupportedAPIException(PortagePackageException):
return _unicode_decode(msg, encoding=_encodings["content"], errors="replace")
+class GPGException(PortageException):
+ """GPG operation failed"""
+
+
class SignatureException(PortageException):
"""Signature was not present in the checked file"""
diff --git a/lib/portage/gpg.py b/lib/portage/gpg.py
new file mode 100644
index 000000000..57be2ebc0
--- /dev/null
+++ b/lib/portage/gpg.py
@@ -0,0 +1,106 @@
+# Copyright 2001-2020 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import subprocess
+import sys
+import threading
+import time
+
+from portage import os
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
+from portage.exception import GPGException
+from portage.output import colorize
+from portage.util import shlex_split, varexpand, writemsg, writemsg_stdout
+
+
+class GPG:
+ """
+ Unlock GPG, must call dircetly from main program for get correct TTY
+ """
+
+ def __init__(self, settings):
+ """
+ Portage settings are needed to run GPG unlock command.
+ """
+ self.settings = settings
+ self.thread = None
+ self.GPG_signing_base_command = self.settings.get(
+ "BINPKG_GPG_SIGNING_BASE_COMMAND"
+ )
+ self.digest_algo = self.settings.get("BINPKG_GPG_SIGNING_DIGEST")
+ self.signing_gpg_home = self.settings.get("BINPKG_GPG_SIGNING_GPG_HOME")
+ self.signing_gpg_key = self.settings.get("BINPKG_GPG_SIGNING_KEY")
+ self.GPG_unlock_command = self.GPG_signing_base_command.replace(
+ "[PORTAGE_CONFIG]",
+ f"--homedir {self.signing_gpg_home} "
+ f"--digest-algo {self.digest_algo} "
+ f"--local-user {self.signing_gpg_key} "
+ "--output /dev/null /dev/null",
+ )
+
+ if "gpg-keepalive" in self.settings.features:
+ self.keepalive = True
+ else:
+ self.keepalive = False
+
+ def unlock(self):
+ """
+ Set GPG_TTY and run GPG unlock command.
+ If gpg-keepalive is set, start keepalive thread.
+ """
+ if self.GPG_unlock_command and (
+ self.settings.get("BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0])
+ == "gpkg"
+ ):
+ try:
+ os.environ["GPG_TTY"] = os.ttyname(sys.stdout.fileno())
+ except OSError as e:
+ # When run with no input/output tty, this will fail.
+ # However, if the password is given by command,
+ # GPG does not need to ask password, so can be ignored.
+ writemsg(colorize("WARN", str(e)) + "\n")
+
+ cmd = shlex_split(varexpand(self.GPG_unlock_command, mydict=self.settings))
+ return_code = subprocess.Popen(cmd).wait()
+
+ if return_code == os.EX_OK:
+ writemsg_stdout(colorize("GOOD", "unlocked") + "\n")
+ sys.stdout.flush()
+ else:
+ raise GPGException("GPG unlock failed")
+
+ if self.keepalive:
+ self.GPG_unlock_command = shlex_split(
+ varexpand(self.GPG_unlock_command, mydict=self.settings)
+ )
+ self.thread = threading.Thread(target=self.gpg_keepalive, daemon=True)
+ self.thread.start()
+
+ def stop(self):
+ """
+ Stop keepalive thread.
+ """
+ if self.thread is not None:
+ self.keepalive = False
+
+ def gpg_keepalive(self):
+ """
+ Call GPG unlock command every 5 mins to avoid the passphrase expired.
+ """
+ count = 0
+ while self.keepalive:
+ if count < 5:
+ time.sleep(60)
+ count += 1
+ continue
+ else:
+ count = 0
+
+ proc = subprocess.Popen(
+ self.GPG_unlock_command,
+ stdin=subprocess.DEVNULL,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.STDOUT,
+ )
+ if proc.wait() != os.EX_OK:
+ raise GPGException("GPG keepalive failed")
diff --git a/lib/portage/gpkg.py b/lib/portage/gpkg.py
new file mode 100644
index 000000000..b642e74ec
--- /dev/null
+++ b/lib/portage/gpkg.py
@@ -0,0 +1,2015 @@
+# Copyright 2001-2020 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import tarfile
+import io
+import threading
+import subprocess
+import errno
+import pwd
+import grp
+import stat
+import sys
+import tempfile
+from copy import copy
+from datetime import datetime
+
+from portage import checksum
+from portage import os
+from portage import shutil
+from portage import normalize_path
+from portage import _encodings
+from portage import _unicode_decode
+from portage import _unicode_encode
+from portage.exception import (
+ FileNotFound,
+ InvalidBinaryPackageFormat,
+ InvalidCompressionMethod,
+ CompressorNotFound,
+ CompressorOperationFailed,
+ CommandNotFound,
+ GPGException,
+ DigestException,
+ MissingSignature,
+ InvalidSignature,
+)
+from portage.output import colorize
+from portage.util._urlopen import urlopen
+from portage.util import writemsg
+from portage.util import shlex_split, varexpand
+from portage.util.compression_probe import _compressors
+from portage.process import find_binary
+from portage.const import MANIFEST2_HASH_DEFAULTS, HASHING_BLOCKSIZE
+
+
+class tar_stream_writer:
+ """
+ One-pass helper function that return a file-like object
+ for create a file inside of a tar container.
+
+ This helper allowed streaming add a new file to tar
+ without prior knows the file size.
+
+ With optional call and pipe data through external program,
+ the helper can transparently save compressed data.
+
+ With optional checksum helper, this helper can create
+ corresponding checksum and GPG signature.
+
+ Example:
+
+ writer = tar_stream_writer(
+ file_tarinfo, # the file tarinfo that need to be added
+ container, # the outer container tarfile object
+ tarfile.USTAR_FORMAT, # the outer container format
+ ["gzip"], # compression command
+ checksum_helper # checksum helper
+ )
+
+ writer.write(data)
+ writer.close()
+ """
+
+ def __init__(
+ self,
+ tarinfo,
+ container,
+ tar_format,
+ cmd=None,
+ checksum_helper=None,
+ uid=None,
+ gid=None,
+ ):
+ """
+ tarinfo # the file tarinfo that need to be added
+ container # the outer container tarfile object
+ tar_format # the outer container format for create the tar header
+ cmd # subprocess.Popen format compression command
+ checksum_helper # checksum helper
+ uid # drop root user to uid
+ gid # drop root group to gid
+ """
+ self.checksum_helper = checksum_helper
+ self.closed = False
+ self.container = container
+ self.killed = False
+ self.tar_format = tar_format
+ self.tarinfo = tarinfo
+ self.uid = uid
+ self.gid = gid
+
+ # Record container end position
+ self.container.fileobj.seek(0, io.SEEK_END)
+ self.begin_position = self.container.fileobj.tell()
+ self.end_position = 0
+ self.file_size = 0
+
+ # Write tar header without size
+ tar_header = self.tarinfo.tobuf(
+ self.tar_format, self.container.encoding, self.container.errors
+ )
+ self.header_size = len(tar_header)
+ self.container.fileobj.write(tar_header)
+ self.container.fileobj.flush()
+ self.container.offset += self.header_size
+
+ # Start external compressor if needed
+ if cmd is None:
+ self.proc = None
+ else:
+ if sys.hexversion >= 0x03090000:
+ self.proc = subprocess.Popen(
+ cmd,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ user=self.uid,
+ group=self.gid,
+ )
+ else:
+ self.proc = subprocess.Popen(
+ cmd,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ preexec_fn=self._drop_privileges,
+ )
+
+ self.read_thread = threading.Thread(
+ target=self._cmd_read_thread, name="tar_stream_cmd_read", daemon=True
+ )
+ self.read_thread.start()
+
+ def __del__(self):
+ self.close()
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.close()
+
+ def _drop_privileges(self):
+ if self.uid:
+ try:
+ os.setuid(self.uid)
+ except PermissionError:
+ writemsg(
+ colorize(
+ "BAD", f"!!! Drop root privileges to user {self.uid} failed."
+ )
+ )
+ raise
+
+ if self.gid:
+ try:
+ os.setgid(self.gid)
+ except PermissionError:
+ writemsg(
+ colorize(
+ "BAD", f"!!! Drop root privileges to group {self.gid} failed."
+ )
+ )
+ raise
+
+ def kill(self):
+ """
+ kill external program if any error happened in python
+ """
+ if self.proc is not None:
+ self.killed = True
+ self.proc.kill()
+ self.proc.stdin.close()
+ self.close()
+
+ def _cmd_read_thread(self):
+ """
+ Use thread to avoid block.
+ Read stdout from external compressor, then write to the file
+ in container, and to checksum helper if needed.
+ """
+ while True:
+ try:
+ buffer = self.proc.stdout.read(HASHING_BLOCKSIZE)
+ if not buffer:
+ self.proc.stdout.close()
+ self.proc.stderr.close()
+ return
+ except BrokenPipeError:
+ self.proc.stdout.close()
+ if not self.killed:
+ # Do not raise error if killed by portage
+ raise CompressorOperationFailed("PIPE broken")
+
+ self.container.fileobj.write(buffer)
+ if self.checksum_helper:
+ self.checksum_helper.update(buffer)
+
+ def write(self, data):
+ """
+ Write data to tarfile or external compressor stdin
+ """
+ if self.closed:
+ raise OSError("writer closed")
+
+ if self.proc:
+ # Write to external program
+ self.proc.stdin.write(data)
+ else:
+ # Write to container
+ self.container.fileobj.write(data)
+ if self.checksum_helper:
+ self.checksum_helper.update(data)
+
+ def close(self):
+ """
+ Update the new file tar header when close
+ """
+ if self.closed:
+ return
+
+ # Wait compressor exit
+ if self.proc is not None:
+ self.proc.stdin.close()
+ if self.proc.wait() != os.EX_OK:
+ raise CompressorOperationFailed("compression failed")
+ if self.read_thread.is_alive():
+ self.read_thread.join()
+
+ # Get container end position and calculate file size
+ self.container.fileobj.seek(0, io.SEEK_END)
+ self.end_position = self.container.fileobj.tell()
+ self.file_size = self.end_position - self.begin_position - self.header_size
+ self.tarinfo.size = self.file_size
+
+ # Tar block is 512, need padding \0
+ _, remainder = divmod(self.file_size, 512)
+ if remainder > 0:
+ padding_size = 512 - remainder
+ self.container.fileobj.write(b"\0" * padding_size)
+ self.container.offset += padding_size
+ self.container.fileobj.flush()
+
+ # Update tar header
+ tar_header = self.tarinfo.tobuf(
+ self.tar_format, self.container.encoding, self.container.errors
+ )
+ self.container.fileobj.seek(self.begin_position)
+ self.container.fileobj.write(tar_header)
+ self.container.fileobj.seek(0, io.SEEK_END)
+ self.container.fileobj.flush()
+ self.container.offset = self.container.fileobj.tell()
+ self.closed = True
+
+ # Add tarinfo to tarfile
+ self.container.members.append(self.tarinfo)
+
+ if self.checksum_helper:
+ self.checksum_helper.finish()
+
+ self.closed = True
+
+
+class tar_stream_reader:
+ """
+ helper function that return a file-like object
+ for read a file inside of a tar container.
+
+ This helper allowed transparently streaming read a compressed
+ file in tar.
+
+ With optional call and pipe compressed data through external
+ program, and return the uncompressed data.
+
+ reader = tar_stream_reader(
+ fileobj, # the fileobj from tarfile.extractfile(f)
+ ["gzip", "-d"], # decompression command
+ )
+
+ reader.read()
+ reader.close()
+ """
+
+ def __init__(self, fileobj, cmd=None, uid=None, gid=None):
+ """
+ fileobj should be a file-like object that have read().
+ cmd is optional external decompressor command.
+ """
+ self.closed = False
+ self.cmd = cmd
+ self.fileobj = fileobj
+ self.killed = False
+ self.uid = uid
+ self.gid = gid
+
+ if cmd is None:
+ self.read_io = fileobj
+ self.proc = None
+ else:
+ # Start external decompressor
+ if sys.hexversion >= 0x03090000:
+ self.proc = subprocess.Popen(
+ cmd,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ user=self.uid,
+ group=self.gid,
+ )
+ else:
+ self.proc = subprocess.Popen(
+ cmd,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ preexec_fn=self._drop_privileges,
+ )
+ self.read_io = self.proc.stdout
+ # Start stdin block writing thread
+ self.thread = threading.Thread(
+ target=self._write_thread, name="tar_stream_stdin_writer", daemon=True
+ )
+ self.thread.start()
+
+ def __del__(self):
+ try:
+ self.close()
+ except CompressorOperationFailed:
+ pass
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ try:
+ self.close()
+ except CompressorOperationFailed:
+ pass
+
+ def _write_thread(self):
+ """
+ writing thread to avoid full buffer blocking
+ """
+ try:
+ while True:
+ buffer = self.fileobj.read(HASHING_BLOCKSIZE)
+ if buffer:
+ try:
+ self.proc.stdin.write(buffer)
+ except ValueError:
+ if self.killed:
+ return
+ else:
+ raise
+ else:
+ self.proc.stdin.flush()
+ self.proc.stdin.close()
+ break
+ except BrokenPipeError:
+ if self.killed is False:
+ raise CompressorOperationFailed("PIPE broken")
+
+ def _drop_privileges(self):
+ if self.uid:
+ try:
+ os.setuid(self.uid)
+ except PermissionError:
+ writemsg(
+ colorize(
+ "BAD", f"!!! Drop root privileges to user {self.uid} failed."
+ )
+ )
+ raise
+
+ if self.gid:
+ try:
+ os.setgid(self.gid)
+ except PermissionError:
+ writemsg(
+ colorize(
+ "BAD", f"!!! Drop root privileges to group {self.gid} failed."
+ )
+ )
+ raise
+
+ def kill(self):
+ """
+ kill external program if any error happened in python
+ """
+ if self.proc is not None:
+ self.killed = True
+ self.proc.kill()
+ self.proc.stdin.close()
+ self.close()
+
+ def read(self, bufsize=-1):
+ """
+ return decompressor stdout data
+ """
+ if self.closed:
+ raise OSError("writer closed")
+ else:
+ return self.read_io.read(bufsize)
+
+ def close(self):
+ """
+ wait external program complete and do clean up
+ """
+ if self.closed:
+ return
+
+ self.closed = True
+
+ if self.proc is not None:
+ self.thread.join()
+ try:
+ if self.proc.wait() != os.EX_OK:
+ if not self.proc.stderr.closed:
+ stderr = self.proc.stderr.read().decode()
+ if not self.killed:
+ writemsg(colorize("BAD", "!!!" + "\n" + stderr))
+ raise CompressorOperationFailed("decompression failed")
+ finally:
+ self.proc.stdout.close()
+ self.proc.stderr.close()
+
+
+class checksum_helper:
+ """
+ Do checksum generation and GPG Signature generation and verification
+ """
+
+ SIGNING = 0
+ VERIFY = 1
+
+ def __init__(self, settings, gpg_operation=None, detached=True, signature=None):
+ """
+ settings # portage settings
+ gpg_operation # either SIGNING or VERIFY
+ signature # GPG signature string used for GPG verify only
+ """
+ self.settings = settings
+ self.gpg_operation = gpg_operation
+ self.gpg_proc = None
+ self.gpg_result = None
+ self.gpg_output = None
+ self.finished = False
+ self.sign_file_path = None
+
+ if (gpg_operation == checksum_helper.VERIFY) and (os.getuid() == 0):
+ try:
+ drop_user = self.settings.get("GPG_VERIFY_USER_DROP", "nobody")
+ if drop_user == "":
+ self.uid = None
+ else:
+ self.uid = pwd.getpwnam(drop_user).pw_uid
+ except KeyError:
+ writemsg(colorize("BAD", f"!!! Failed to find user {drop_user}."))
+ raise
+
+ try:
+ drop_group = self.settings.get("GPG_VERIFY_GROUP_DROP", "nogroup")
+ if drop_group == "":
+ self.gid = None
+ else:
+ self.gid = grp.getgrnam(drop_group).gr_gid
+ except KeyError:
+ writemsg(colorize("BAD", f"!!! Failed to find group {drop_group}."))
+ raise
+ else:
+ self.uid = None
+ self.gid = None
+
+ # Initialize the hash libs
+ self.libs = {}
+ for hash_name in MANIFEST2_HASH_DEFAULTS:
+ self.libs[hash_name] = checksum.hashfunc_map[hash_name]._hashobject()
+
+ # GPG
+ env = self.settings.environ()
+ if self.gpg_operation == checksum_helper.SIGNING:
+ gpg_signing_base_command = self.settings.get(
+ "BINPKG_GPG_SIGNING_BASE_COMMAND"
+ )
+ digest_algo = self.settings.get("BINPKG_GPG_SIGNING_DIGEST")
+ gpg_home = self.settings.get("BINPKG_GPG_SIGNING_GPG_HOME")
+ gpg_key = self.settings.get("BINPKG_GPG_SIGNING_KEY")
+
+ if detached:
+ gpg_detached = "--detach-sig"
+ else:
+ gpg_detached = "--clear-sign"
+
+ if gpg_signing_base_command:
+ gpg_signing_command = gpg_signing_base_command.replace(
+ "[PORTAGE_CONFIG]",
+ f"--homedir {gpg_home} "
+ f"--digest-algo {digest_algo} "
+ f"--local-user {gpg_key} "
+ f"{gpg_detached} "
+ "--batch --no-tty",
+ )
+
+ gpg_signing_command = shlex_split(
+ varexpand(gpg_signing_command, mydict=self.settings)
+ )
+ gpg_signing_command = [x for x in gpg_signing_command if x != ""]
+ try:
+ env["GPG_TTY"] = os.ttyname(sys.stdout.fileno())
+ except OSError:
+ pass
+ else:
+ raise CommandNotFound("GPG signing command is not set")
+
+ self.gpg_proc = subprocess.Popen(
+ gpg_signing_command,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env=env,
+ )
+
+ elif self.gpg_operation == checksum_helper.VERIFY:
+ if (signature is None) and (detached == True):
+ raise MissingSignature("No signature provided")
+
+ gpg_verify_base_command = self.settings.get(
+ "BINPKG_GPG_VERIFY_BASE_COMMAND"
+ )
+ gpg_home = self.settings.get("BINPKG_GPG_VERIFY_GPG_HOME")
+
+ if not gpg_verify_base_command:
+ raise CommandNotFound("GPG verify command is not set")
+
+ gpg_verify_command = gpg_verify_base_command.replace(
+ "[PORTAGE_CONFIG]", f"--homedir {gpg_home} "
+ )
+
+ if detached:
+ self.sign_file_fd, self.sign_file_path = tempfile.mkstemp(
+ ".sig", "portage-sign-"
+ )
+
+ gpg_verify_command = gpg_verify_command.replace(
+ "[SIGNATURE]", f"{self.sign_file_path} -"
+ )
+
+ # Create signature file and allow everyone read
+ with open(self.sign_file_fd, "wb") as sign:
+ sign.write(signature)
+ os.chmod(self.sign_file_path, 0o644)
+ else:
+ gpg_verify_command = gpg_verify_command.replace(
+ "[SIGNATURE]", "--output - -"
+ )
+
+ gpg_verify_command = shlex_split(
+ varexpand(gpg_verify_command, mydict=self.settings)
+ )
+ gpg_verify_command = [x for x in gpg_verify_command if x != ""]
+
+ if sys.hexversion >= 0x03090000:
+ self.gpg_proc = subprocess.Popen(
+ gpg_verify_command,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env=env,
+ user=self.uid,
+ group=self.gid,
+ )
+
+ else:
+ self.gpg_proc = subprocess.Popen(
+ gpg_verify_command,
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env=env,
+ preexec_fn=self._drop_privileges,
+ )
+
+ def __del__(self):
+ self.finish()
+
+ def _check_gpg_status(self, gpg_status):
+ """
+ Check GPG status log for extra info.
+ GPG will return OK even if the signature owner is not trusted.
+ """
+ good_signature = False
+ trust_signature = False
+
+ for l in gpg_status.splitlines():
+ if l.startswith("[GNUPG:] GOODSIG"):
+ good_signature = True
+
+ if l.startswith("[GNUPG:] TRUST_ULTIMATE") or l.startswith(
+ "[GNUPG:] TRUST_FULLY"
+ ):
+ trust_signature = True
+
+ if (not good_signature) or (not trust_signature):
+ writemsg(colorize("BAD", "!!!" + "\n" + self.gpg_result.decode()))
+ raise InvalidSignature("GPG verify failed")
+
+ def _drop_privileges(self):
+ if self.uid:
+ try:
+ os.setuid(self.uid)
+ except PermissionError:
+ writemsg(
+ colorize(
+ "BAD", f"!!! Drop root privileges to user {self.uid} failed."
+ )
+ )
+ raise
+
+ if self.gid:
+ try:
+ os.setgid(self.gid)
+ except PermissionError:
+ writemsg(
+ colorize(
+ "BAD", f"!!! Drop root privileges to group {self.gid} failed."
+ )
+ )
+ raise
+
+ def update(self, data):
+ """
+ Write data to hash libs and GPG stdin.
+ """
+ for c in self.libs:
+ self.libs[c].update(data)
+
+ if self.gpg_proc is not None:
+ self.gpg_proc.stdin.write(data)
+
+ def finish(self):
+ """
+ Tell GPG file is EOF, and get results, then do clean up.
+ """
+ if self.finished:
+ return
+
+ if self.gpg_proc is not None:
+ # Tell GPG EOF
+ self.gpg_proc.stdin.close()
+
+ return_code = self.gpg_proc.wait()
+
+ if self.sign_file_path:
+ os.remove(self.sign_file_path)
+
+ self.finished = True
+
+ self.gpg_result = self.gpg_proc.stderr.read()
+ self.gpg_output = self.gpg_proc.stdout.read()
+ self.gpg_proc.stdout.close()
+ self.gpg_proc.stderr.close()
+
+ if return_code == os.EX_OK:
+ if self.gpg_operation == checksum_helper.VERIFY:
+ self._check_gpg_status(self.gpg_result.decode())
+ else:
+ writemsg(colorize("BAD", "!!!" + "\n" + self.gpg_result.decode()))
+ if self.gpg_operation == checksum_helper.SIGNING:
+ writemsg(colorize("BAD", self.gpg_output.decode()))
+ raise GPGException("GPG signing failed")
+ elif self.gpg_operation == checksum_helper.VERIFY:
+ raise InvalidSignature("GPG verify failed")
+
+
+class tar_safe_extract:
+ """
+ A safer version of tar extractall that doing sanity check.
+ Note that this does not solve all security problems.
+ """
+
+ def __init__(self, tar: tarfile.TarFile, prefix: str = ""):
+ """
+ tar: an opened TarFile that ready to be read.
+ prefix: a optional prefix for an inner directory should be considered
+ as the root directory. e.g. "metadata" and "image".
+ """
+ self.tar = tar
+ self.prefix = prefix
+ self.closed = False
+ self.file_list = []
+
+ def extractall(self, dest_dir: str):
+ """
+ Extract all files to a temporary directory in the dest_dir, and move
+ them to the dest_dir after sanity check.
+ """
+ if self.closed:
+ raise IOError("Tar file is closed.")
+ temp_dir = tempfile.TemporaryDirectory(dir=dest_dir)
+ try:
+ while True:
+ member = self.tar.next()
+ if member is None:
+ break
+ if (member.name in self.file_list) or (
+ os.path.join(".", member.name) in self.file_list
+ ):
+ writemsg(
+ colorize(
+ "BAD", f"Danger: duplicate files detected: {member.name}"
+ )
+ )
+ raise ValueError("Duplicate files detected.")
+ if member.name.startswith("/"):
+ writemsg(
+ colorize(
+ "BAD", f"Danger: absolute path detected: {member.name}"
+ )
+ )
+ raise ValueError("Absolute path detected.")
+ if member.name.startswith("../") or ("/../" in member.name):
+ writemsg(
+ colorize(
+ "BAD", f"Danger: path traversal detected: {member.name}"
+ )
+ )
+ raise ValueError("Path traversal detected.")
+ if member.isdev():
+ writemsg(
+ colorize("BAD", f"Danger: device file detected: {member.name}")
+ )
+ raise ValueError("Device file detected.")
+ if member.islnk() and (member.linkname not in self.file_list):
+ writemsg(
+ colorize(
+ "BAD", f"Danger: hardlink escape detected: {member.name}"
+ )
+ )
+ raise ValueError("Hardlink escape detected.")
+
+ self.file_list.append(member.name)
+ self.tar.extract(member, path=temp_dir.name)
+
+ data_dir = os.path.join(temp_dir.name, self.prefix)
+ for file in os.listdir(data_dir):
+ shutil.move(os.path.join(data_dir, file), os.path.join(dest_dir, file))
+ finally:
+ temp_dir.cleanup()
+ self.closed = True
+
+
+class gpkg:
+ """
+ Gentoo binary package
+ https://www.gentoo.org/glep/glep-0078.html
+ """
+
+ def __init__(self, settings, base_name=None, gpkg_file=None):
+ """
+ gpkg class handle all gpkg operations for one package.
+ base_name is the package basename.
+ gpkg_file should be exists file path for read or will create.
+ """
+ if sys.version_info.major < 3:
+ raise InvalidBinaryPackageFormat("GPKG not support Python 2")
+ self.settings = settings
+ self.gpkg_version = "gpkg-1"
+ if gpkg_file is None:
+ self.gpkg_file = None
+ else:
+ self.gpkg_file = _unicode_decode(
+ gpkg_file, encoding=_encodings["fs"], errors="strict"
+ )
+ self.base_name = base_name
+ self.checksums = []
+ self.manifest_old = []
+
+ # Compression is the compression algorithm, if set to None will
+ # not use compression.
+ self.compression = self.settings.get("BINPKG_COMPRESS", None)
+ if self.compression in ["", "none"]:
+ self.compression = None
+
+ # The create_signature is whether create signature for the package or not.
+ if "binpkg-signing" in self.settings.features:
+ self.create_signature = True
+ else:
+ self.create_signature = False
+
+ # The request_signature is whether signature files are mandatory.
+ # If set true, any missing signature file will cause reject processing.
+ if "binpkg-request-signature" in self.settings.features:
+ self.request_signature = True
+ else:
+ self.request_signature = False
+
+ # The verify_signature is whether verify package signature or not.
+ # In rare case user may want to ignore signature,
+ # E.g. package with expired signature.
+ if "binpkg-ignore-signature" in self.settings.features:
+ self.verify_signature = False
+ else:
+ self.verify_signature = True
+
+ self.ext_list = {
+ "gzip": ".gz",
+ "bzip2": ".bz2",
+ "lz4": ".lz4",
+ "lzip": ".lz",
+ "lzop": ".lzo",
+ "xz": ".xz",
+ "zstd": ".zst",
+ }
+
+ def unpack_metadata(self, dest_dir=None):
+ """
+ Unpack metadata to dest_dir.
+ If dest_dir is None, return files and values in dict.
+ The dict key will be UTF-8, not bytes.
+ """
+ self._verify_binpkg(metadata_only=True)
+
+ with tarfile.open(self.gpkg_file, "r") as container:
+ metadata_tarinfo, metadata_comp = self._get_inner_tarinfo(
+ container, "metadata"
+ )
+
+ with tar_stream_reader(
+ container.extractfile(metadata_tarinfo),
+ self._get_decompression_cmd(metadata_comp),
+ ) as metadata_reader:
+ metadata_tar = io.BytesIO(metadata_reader.read())
+
+ with tarfile.open(mode="r:", fileobj=metadata_tar) as metadata:
+ if dest_dir is None:
+ metadata_ = {
+ os.path.relpath(k.name, "metadata"): metadata.extractfile(
+ k
+ ).read()
+ for k in metadata.getmembers()
+ }
+ else:
+ metadata_safe = tar_safe_extract(metadata, "metadata")
+ metadata_safe.extractall(dest_dir)
+ metadata_ = True
+ metadata_tar.close()
+ return metadata_
+
+ def get_metadata(self, want=None):
+ """
+ get package metadata.
+ if want is list, return all want key-values in dict
+ if want is str, return the want key value
+ """
+ if want is None:
+ return self.unpack_metadata()
+ elif isinstance(want, str):
+ metadata = self.unpack_metadata()
+ metadata_want = metadata.get(want, None)
+ return metadata_want
+ else:
+ metadata = self.unpack_metadata()
+ metadata_want = {k: metadata.get(k, None) for k in want}
+ return metadata_want
+
+ def get_metadata_url(self, url, want=None):
+ """
+ Return the requested metadata from url gpkg.
+ Default return all meta data.
+ Use 'want' to get specific name from metadata.
+ This method only support the correct package format.
+ Wrong files order or incorrect basename will be considered invalid
+ to reduce potential attacks.
+ Only signature will be check if the signature file is the next file.
+ Manifest will be ignored since it will be at the end of package.
+ """
+ # The init download file head size
+ init_size = 51200
+
+ # Load remote container
+ container_file = io.BytesIO(
+ urlopen(url, headers={"Range": "bytes=0-" + str(init_size)}).read()
+ )
+
+ # Check gpkg and metadata
+ with tarfile.open(mode="r", fileobj=container_file) as container:
+ if self.gpkg_version not in container.getnames():
+ raise InvalidBinaryPackageFormat("Invalid gpkg file.")
+
+ metadata_tarinfo, metadata_comp = self._get_inner_tarinfo(
+ container, "metadata"
+ )
+
+ # Extra 10240 bytes for signature
+ end_size = metadata_tarinfo.offset_data + metadata_tarinfo.size + 10240
+ _, remainder = divmod(end_size, 512)
+ end_size += 512 - remainder
+
+ # If need more data
+ if end_size > 10000000:
+ raise InvalidBinaryPackageFormat("metadata too large " + str(end_size))
+ if end_size > init_size:
+ container_file.seek(0, io.SEEK_END)
+ container_file.write(
+ urlopen(
+ url,
+ headers={
+ "Range": "bytes=" + str(init_size + 1) + "-" + str(end_size)
+ },
+ ).read()
+ )
+
+ container_file.seek(0)
+
+ # Reload and process full metadata
+ with tarfile.open(mode="r", fileobj=container_file) as container:
+ metadata_tarinfo, metadata_comp = self._get_inner_tarinfo(
+ container, "metadata"
+ )
+
+ # Verify metadata file signature if needed
+ # binpkg-ignore-signature can override this.
+ signature_filename = metadata_tarinfo.name + ".sig"
+ if signature_filename in container.getnames():
+ if self.request_signature and self.verify_signature:
+ metadata_signature = container.extractfile(
+ signature_filename
+ ).read()
+ checksum_info = checksum_helper(
+ self.settings,
+ gpg_operation=checksum_helper.VERIFY,
+ signature=metadata_signature,
+ )
+ checksum_info.update(container.extractfile(metadata_tarinfo).read())
+ checksum_info.finish()
+
+ # Load metadata
+ with tar_stream_reader(
+ container.extractfile(metadata_tarinfo),
+ self._get_decompression_cmd(metadata_comp),
+ ) as metadata_reader:
+ metadata_file = io.BytesIO(metadata_reader.read())
+
+ with tarfile.open(mode="r:", fileobj=metadata_file) as metadata:
+ if want is None:
+ metadata_ = {
+ os.path.relpath(k.name, "metadata"): metadata.extractfile(
+ k
+ ).read()
+ for k in metadata.getmembers()
+ }
+ else:
+ metadata_ = {
+ os.path.relpath(k.name, "metadata"): metadata.extractfile(
+ k
+ ).read()
+ for k in metadata.getmembers()
+ if k in want
+ }
+ metadata_file.close()
+ container_file.close()
+ return metadata_
+
+ def compress(self, root_dir, metadata, clean=False):
+ """
+ Use initialized configuation create new gpkg file from root_dir.
+ Will overwrite any exists file.
+ metadata is a dict, the key will be file name, the value will be
+ the file contents.
+ """
+
+ root_dir = normalize_path(
+ _unicode_decode(root_dir, encoding=_encodings["fs"], errors="strict")
+ )
+
+ # Get pre image info
+ container_tar_format, image_tar_format = self._get_tar_format_from_stats(
+ *self._check_pre_image_files(root_dir)
+ )
+
+ # Long CPV
+ if len(self.base_name) >= 154:
+ container_tar_format = tarfile.GNU_FORMAT
+
+ # gpkg container
+ container = tarfile.TarFile(
+ name=self.gpkg_file, mode="w", format=container_tar_format
+ )
+
+ # gpkg version
+ gpkg_version_file = tarfile.TarInfo(self.gpkg_version)
+ gpkg_version_file.mtime = datetime.utcnow().timestamp()
+ container.addfile(gpkg_version_file)
+
+ compression_cmd = self._get_compression_cmd()
+
+ # metadata
+ self._add_metadata(container, metadata, compression_cmd)
+
+ # image
+ if self.create_signature:
+ checksum_info = checksum_helper(
+ self.settings, gpg_operation=checksum_helper.SIGNING
+ )
+ else:
+ checksum_info = checksum_helper(self.settings)
+
+ image_tarinfo = self._create_tarinfo("image")
+ image_tarinfo.mtime = datetime.utcnow().timestamp()
+ with tar_stream_writer(
+ image_tarinfo, container, image_tar_format, compression_cmd, checksum_info
+ ) as image_writer:
+ with tarfile.open(
+ mode="w|", fileobj=image_writer, format=image_tar_format
+ ) as image_tar:
+ image_tar.add(root_dir, "image", recursive=True)
+
+ image_tarinfo = container.getmember(image_tarinfo.name)
+ self._record_checksum(checksum_info, image_tarinfo)
+
+ if self.create_signature:
+ self._add_signature(checksum_info, image_tarinfo, container)
+
+ self._add_manifest(container)
+ container.close()
+
+ def decompress(self, decompress_dir):
+ """
+ decompress current gpkg to decompress_dir
+ """
+ decompress_dir = normalize_path(
+ _unicode_decode(decompress_dir, encoding=_encodings["fs"], errors="strict")
+ )
+
+ self._verify_binpkg()
+ os.makedirs(decompress_dir, mode=0o755, exist_ok=True)
+
+ with tarfile.open(self.gpkg_file, "r") as container:
+ image_tarinfo, image_comp = self._get_inner_tarinfo(container, "image")
+
+ with tar_stream_reader(
+ container.extractfile(image_tarinfo),
+ self._get_decompression_cmd(image_comp),
+ ) as image_tar:
+
+ with tarfile.open(mode="r|", fileobj=image_tar) as image:
+ try:
+ image_safe = tar_safe_extract(image, "image")
+ image_safe.extractall(decompress_dir)
+ except Exception as ex:
+ writemsg(colorize("BAD", "!!!" + "Extract failed."))
+ raise
+ finally:
+ image_tar.kill()
+
+ def update_metadata(self, metadata, newcpv=None):
+ """
+ Update metadata in the gpkg file.
+ """
+ self._verify_binpkg()
+ self.checksums = []
+ oldcpv = None
+
+ if newcpv:
+ oldcpv = self.base_name
+
+ with open(self.gpkg_file, "rb") as container:
+ container_tar_format = self._get_tar_format(container)
+ if container_tar_format is None:
+ raise InvalidBinaryPackageFormat("Cannot identify tar format")
+
+ # container
+ tmp_gpkg_file_name = self.gpkg_file + "." + str(os.getpid())
+ with tarfile.TarFile(
+ name=tmp_gpkg_file_name, mode="w", format=container_tar_format
+ ) as container:
+ # gpkg version
+ gpkg_version_file = tarfile.TarInfo(self.gpkg_version)
+ gpkg_version_file.mtime = datetime.utcnow().timestamp()
+ container.addfile(gpkg_version_file)
+
+ compression_cmd = self._get_compression_cmd()
+
+ # metadata
+ if newcpv:
+ self.base_name = newcpv
+ self._add_metadata(container, metadata, compression_cmd)
+ self.base_name = oldcpv
+ else:
+ self._add_metadata(container, metadata, compression_cmd)
+
+ # reuse other stuffs
+ with tarfile.open(self.gpkg_file, "r") as container_old:
+ manifest_old = self.manifest_old.copy()
+
+ for m in manifest_old:
+ file_name_old = m[1]
+ if os.path.basename(file_name_old).startswith("metadata"):
+ continue
+ old_data_tarinfo = container_old.getmember(file_name_old)
+ new_data_tarinfo = copy(old_data_tarinfo)
+ if newcpv:
+ m[1] = m[1].replace(oldcpv, newcpv, 1)
+ new_data_tarinfo.name = new_data_tarinfo.name.replace(
+ oldcpv, newcpv, 1
+ )
+ container.addfile(
+ new_data_tarinfo, container_old.extractfile(old_data_tarinfo)
+ )
+ self.checksums.append(m)
+
+ self._add_manifest(container)
+
+ shutil.move(tmp_gpkg_file_name, self.gpkg_file)
+
+ def _add_metadata(self, container, metadata, compression_cmd):
+ """
+ add metadata to container
+ """
+ if metadata is None:
+ metadata = {}
+ metadata_tarinfo = self._create_tarinfo("metadata")
+ metadata_tarinfo.mtime = datetime.utcnow().timestamp()
+
+ if self.create_signature:
+ checksum_info = checksum_helper(
+ self.settings, gpg_operation=checksum_helper.SIGNING
+ )
+ else:
+ checksum_info = checksum_helper(self.settings)
+
+ with tar_stream_writer(
+ metadata_tarinfo,
+ container,
+ tarfile.USTAR_FORMAT,
+ compression_cmd,
+ checksum_info,
+ ) as metadata_writer:
+ with tarfile.open(
+ mode="w|", fileobj=metadata_writer, format=tarfile.USTAR_FORMAT
+ ) as metadata_tar:
+
+ for m in metadata:
+ m_info = tarfile.TarInfo(os.path.join("metadata", m))
+ m_info.mtime = datetime.utcnow().timestamp()
+
+ if isinstance(metadata[m], bytes):
+ m_data = io.BytesIO(metadata[m])
+ else:
+ m_data = io.BytesIO(metadata[m].encode("UTF-8"))
+
+ m_data.seek(0, io.SEEK_END)
+ m_info.size = m_data.tell()
+ m_data.seek(0)
+ metadata_tar.addfile(m_info, m_data)
+ m_data.close()
+
+ metadata_tarinfo = container.getmember(metadata_tarinfo.name)
+ self._record_checksum(checksum_info, metadata_tarinfo)
+
+ if self.create_signature:
+ self._add_signature(checksum_info, metadata_tarinfo, container)
+
+ def _quickpkg(self, contents, metadata, root_dir, protect=None):
+ """
+ Similar to compress, but for quickpkg.
+ Will compress the given files to image with root,
+ ignoring all other files.
+ """
+
+ protect_file = io.BytesIO(
+ b"# empty file because --include-config=n " + b"when `quickpkg` was used\n"
+ )
+ protect_file.seek(0, io.SEEK_END)
+ protect_file_size = protect_file.tell()
+
+ root_dir = normalize_path(
+ _unicode_decode(root_dir, encoding=_encodings["fs"], errors="strict")
+ )
+
+ # Get pre image info
+ container_tar_format, image_tar_format = self._get_tar_format_from_stats(
+ *self._check_pre_quickpkg_files(contents, root_dir)
+ )
+
+ # Long CPV
+ if len(self.base_name) >= 154:
+ container_tar_format = tarfile.GNU_FORMAT
+
+ # GPKG container
+ container = tarfile.TarFile(
+ name=self.gpkg_file, mode="w", format=container_tar_format
+ )
+
+ # GPKG version
+ gpkg_version_file = tarfile.TarInfo(self.gpkg_version)
+ gpkg_version_file.mtime = datetime.utcnow().timestamp()
+ container.addfile(gpkg_version_file)
+
+ compression_cmd = self._get_compression_cmd()
+ # Metadata
+ self._add_metadata(container, metadata, compression_cmd)
+
+ # Image
+ if self.create_signature:
+ checksum_info = checksum_helper(
+ self.settings, gpg_operation=checksum_helper.SIGNING
+ )
+ else:
+ checksum_info = checksum_helper(self.settings)
+
+ paths = list(contents)
+ paths.sort()
+ image_tarinfo = self._create_tarinfo("image")
+ image_tarinfo.mtime = datetime.utcnow().timestamp()
+ with tar_stream_writer(
+ image_tarinfo, container, image_tar_format, compression_cmd, checksum_info
+ ) as image_writer:
+ with tarfile.open(
+ mode="w|", fileobj=image_writer, format=image_tar_format
+ ) as image_tar:
+ if len(paths) == 0:
+ tarinfo = image_tar.tarinfo("image")
+ tarinfo.type = tarfile.DIRTYPE
+ tarinfo.size = 0
+ tarinfo.mode = 0o755
+ image_tar.addfile(tarinfo)
+
+ for path in paths:
+ try:
+ lst = os.lstat(path)
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ raise
+ del e
+ continue
+ contents_type = contents[path][0]
+ if path.startswith(root_dir):
+ arcname = "image/" + path[len(root_dir) :]
+ else:
+ raise ValueError("invalid root argument: '%s'" % root_dir)
+ live_path = path
+ if (
+ "dir" == contents_type
+ and not stat.S_ISDIR(lst.st_mode)
+ and os.path.isdir(live_path)
+ ):
+ # Even though this was a directory in the original ${D}, it exists
+ # as a symlink to a directory in the live filesystem. It must be
+ # recorded as a real directory in the tar file to ensure that tar
+ # can properly extract it's children.
+ live_path = os.path.realpath(live_path)
+ lst = os.lstat(live_path)
+
+ # Since os.lstat() inside TarFile.gettarinfo() can trigger a
+ # UnicodeEncodeError when python has something other than utf_8
+ # return from sys.getfilesystemencoding() (as in bug #388773),
+ # we implement the needed functionality here, using the result
+ # of our successful lstat call. An alternative to this would be
+ # to pass in the fileobj argument to TarFile.gettarinfo(), so
+ # that it could use fstat instead of lstat. However, that would
+ # have the unwanted effect of dereferencing symlinks.
+
+ tarinfo = image_tar.tarinfo(arcname)
+ tarinfo.mode = lst.st_mode
+ tarinfo.uid = lst.st_uid
+ tarinfo.gid = lst.st_gid
+ tarinfo.size = 0
+ tarinfo.mtime = lst.st_mtime
+ tarinfo.linkname = ""
+ if stat.S_ISREG(lst.st_mode):
+ inode = (lst.st_ino, lst.st_dev)
+ if (
+ lst.st_nlink > 1
+ and inode in image_tar.inodes
+ and arcname != image_tar.inodes[inode]
+ ):
+ tarinfo.type = tarfile.LNKTYPE
+ tarinfo.linkname = image_tar.inodes[inode]
+ else:
+ image_tar.inodes[inode] = arcname
+ tarinfo.type = tarfile.REGTYPE
+ tarinfo.size = lst.st_size
+ elif stat.S_ISDIR(lst.st_mode):
+ tarinfo.type = tarfile.DIRTYPE
+ elif stat.S_ISLNK(lst.st_mode):
+ tarinfo.type = tarfile.SYMTYPE
+ tarinfo.linkname = os.readlink(live_path)
+ else:
+ continue
+ try:
+ tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0]
+ except KeyError:
+ pass
+ try:
+ tarinfo.gname = grp.getgrgid(tarinfo.gid)[0]
+ except KeyError:
+ pass
+
+ if stat.S_ISREG(lst.st_mode):
+ if protect and protect(path):
+ protect_file.seek(0)
+ tarinfo.size = protect_file_size
+ image_tar.addfile(tarinfo, protect_file)
+ else:
+ path_bytes = _unicode_encode(
+ path, encoding=_encodings["fs"], errors="strict"
+ )
+
+ with open(path_bytes, "rb") as f:
+ image_tar.addfile(tarinfo, f)
+
+ else:
+ image_tar.addfile(tarinfo)
+
+ image_tarinfo = container.getmember(image_tarinfo.name)
+ self._record_checksum(checksum_info, image_tarinfo)
+
+ if self.create_signature:
+ self._add_signature(checksum_info, image_tarinfo, container)
+
+ self._add_manifest(container)
+ container.close()
+
+ def _record_checksum(self, checksum_info, tarinfo):
+ """
+ Record checksum result for the given file.
+ Replace old checksum if already exists.
+ """
+ for c in self.checksums:
+ if c[1] == tarinfo.name:
+ self.checksums.remove(c)
+ break
+
+ checksum_record = ["DATA", tarinfo.name, str(tarinfo.size)]
+
+ for c in checksum_info.libs:
+ checksum_record.append(c)
+ checksum_record.append(checksum_info.libs[c].hexdigest())
+
+ self.checksums.append(checksum_record)
+
+ def _add_manifest(self, container):
+ """
+ Add Manifest to the container based on current checksums.
+ Creare GPG signatue if needed.
+ """
+ manifest = io.BytesIO()
+
+ for m in self.checksums:
+ manifest.write((" ".join(m) + "\n").encode("UTF-8"))
+
+ if self.create_signature:
+ checksum_info = checksum_helper(
+ self.settings, gpg_operation=checksum_helper.SIGNING, detached=False
+ )
+ checksum_info.update(manifest.getvalue())
+ checksum_info.finish()
+ manifest.seek(0)
+ manifest.write(checksum_info.gpg_output)
+
+ manifest_tarinfo = tarfile.TarInfo("Manifest")
+ manifest_tarinfo.size = manifest.tell()
+ manifest_tarinfo.mtime = datetime.utcnow().timestamp()
+ manifest.seek(0)
+ container.addfile(manifest_tarinfo, manifest)
+ manifest.close()
+
+ def _load_manifest(self, manifest_string):
+ """
+ Check, load, and return manifest in a list by files
+ """
+ manifest = []
+ manifest_filenames = []
+
+ for manifest_record in manifest_string.splitlines():
+ if manifest_record == "":
+ continue
+ manifest_record = manifest_record.strip().split()
+
+ if manifest_record[0] != "DATA":
+ raise DigestException("invalied Manifest")
+
+ if manifest_record[1] in manifest_filenames:
+ raise DigestException("Manifest duplicate file exists")
+
+ try:
+ int(manifest_record[2])
+ except ValueError:
+ raise DigestException("Manifest invalied file size")
+
+ manifest.append(manifest_record)
+ manifest_filenames.append(manifest_record[1])
+
+ return manifest
+
+ def _add_signature(self, checksum_info, tarinfo, container, manifest=True):
+ """
+ Add GPG signature for the given tarinfo file.
+ manifest: add to manifest
+ """
+ if checksum_info.gpg_output is None:
+ raise GPGException("GPG signature is not exists")
+
+ signature = io.BytesIO(checksum_info.gpg_output)
+ signature_tarinfo = tarfile.TarInfo(tarinfo.name + ".sig")
+ signature_tarinfo.size = len(signature.getvalue())
+ signature_tarinfo.mtime = datetime.utcnow().timestamp()
+ container.addfile(signature_tarinfo, signature)
+
+ if manifest:
+ signature_checksum_info = checksum_helper(self.settings)
+ signature.seek(0)
+ signature_checksum_info.update(signature.read())
+ signature_checksum_info.finish()
+ self._record_checksum(signature_checksum_info, signature_tarinfo)
+
+ signature.close()
+
+ def _verify_binpkg(self, metadata_only=False):
+ """
+ Verify current GPKG file.
+ """
+ # Check file path
+ if self.gpkg_file is None:
+ raise FileNotFound("no gpkg file provided")
+
+ # Check if is file
+ if not os.path.isfile(self.gpkg_file):
+ raise FileNotFound(f"File not found {self.gpkg_file}")
+
+ # Check if is tar file
+ with open(self.gpkg_file, "rb") as container:
+ container_tar_format = self._get_tar_format(container)
+ if container_tar_format is None:
+ raise InvalidBinaryPackageFormat(
+ f"Cannot identify tar format: {self.gpkg_file}"
+ )
+
+ # Check container
+ with tarfile.open(self.gpkg_file, "r") as container:
+ try:
+ container_files = container.getnames()
+ except tarfile.ReadError:
+ raise InvalidBinaryPackageFormat(
+ f"Cannot read tar file: {self.gpkg_file}"
+ )
+
+ # Check gpkg header
+ if self.gpkg_version not in container_files:
+ raise InvalidBinaryPackageFormat(f"Invalid gpkg file: {self.gpkg_file}")
+
+ # If any signature exists, we assume all files have signature.
+ if any(f.endswith(".sig") for f in container_files):
+ signature_exist = True
+ else:
+ signature_exist = False
+
+ # Check if all files are unique to avoid same name attack
+ container_files_unique = []
+ for f in container_files:
+ if f in container_files_unique:
+ raise InvalidBinaryPackageFormat(
+ "Duplicate file %s exist, potential attack?" % f
+ )
+ container_files_unique.append(f)
+
+ del container_files_unique
+
+ # Add all files to check list
+ unverified_files = container_files.copy()
+ unverified_files.remove(self.gpkg_version)
+
+ # Check Manifest file
+ if "Manifest" not in unverified_files:
+ raise MissingSignature(f"Manifest not found: {self.gpkg_file}")
+
+ manifest_file = container.extractfile("Manifest")
+ manifest_data = manifest_file.read()
+ manifest_file.close()
+
+ if b"-----BEGIN PGP SIGNATURE-----" in manifest_data:
+ signature_exist = True
+
+ # Check Manifest signature if needed.
+ # binpkg-ignore-signature can override this.
+ if self.request_signature or signature_exist:
+ checksum_info = checksum_helper(
+ self.settings, gpg_operation=checksum_helper.VERIFY, detached=False
+ )
+
+ try:
+ checksum_info.update(manifest_data)
+ checksum_info.finish()
+ except (InvalidSignature, MissingSignature):
+ if self.verify_signature:
+ raise
+
+ manifest_data = checksum_info.gpg_output
+ unverified_files.remove("Manifest")
+ else:
+ unverified_files.remove("Manifest")
+
+ # Load manifest and create manifest check list
+ manifest = self._load_manifest(manifest_data.decode("UTF-8"))
+ unverified_manifest = manifest.copy()
+
+ # Check all remaining files
+ for f in unverified_files.copy():
+ if f.endswith(".sig"):
+ f_signature = None
+ else:
+ f_signature = f + ".sig"
+
+ # Find current file manifest record
+ manifest_record = None
+ for m in manifest:
+ if m[1] == f:
+ manifest_record = m
+
+ if manifest_record is None:
+ raise DigestException(f"{f} checksum not found in {self.gpkg_file}")
+
+ if int(manifest_record[2]) != int(container.getmember(f).size):
+ raise DigestException(
+ f"{f} file size mismatched in {self.gpkg_file}"
+ )
+
+ # Ignore image file and signature if not needed
+ if os.path.basename(f).startswith("image") and metadata_only:
+ unverified_files.remove(f)
+ unverified_manifest.remove(manifest_record)
+ continue
+
+ # Verify current file signature if needed
+ # binpkg-ignore-signature can override this.
+ if (
+ (self.request_signature or signature_exist)
+ and self.verify_signature
+ and f_signature
+ ):
+ if f_signature in unverified_files:
+ signature_file = container.extractfile(f_signature)
+ signature = signature_file.read()
+ signature_file.close()
+ checksum_info = checksum_helper(
+ self.settings,
+ gpg_operation=checksum_helper.VERIFY,
+ signature=signature,
+ )
+ else:
+ raise MissingSignature(
+ f"{f} signature not found in {self.gpkg_file}"
+ )
+ else:
+ checksum_info = checksum_helper(self.settings)
+
+ # Verify current file checksum
+ f_io = container.extractfile(f)
+ while True:
+ buffer = f_io.read(HASHING_BLOCKSIZE)
+ if buffer:
+ checksum_info.update(buffer)
+ else:
+ checksum_info.finish()
+ break
+ f_io.close()
+
+ # At least one supported checksum must be checked
+ verified_hash_count = 0
+ for c in checksum_info.libs:
+ try:
+ if (
+ checksum_info.libs[c].hexdigest().lower()
+ == manifest_record[manifest_record.index(c) + 1].lower()
+ ):
+ verified_hash_count += 1
+ else:
+ raise DigestException(
+ f"{f} checksum mismatched in {self.gpkg_file}"
+ )
+ except KeyError:
+ # Checksum method not supported
+ pass
+
+ if verified_hash_count < 1:
+ raise DigestException(
+ f"{f} no supported checksum found in {self.gpkg_file}"
+ )
+
+ # Current file verified
+ unverified_files.remove(f)
+ unverified_manifest.remove(manifest_record)
+
+ # Check if any file IN Manifest but NOT IN binary package
+ if len(unverified_manifest) != 0:
+ raise DigestException(
+ f"Missing files: {str(unverified_manifest)} in {self.gpkg_file}"
+ )
+
+ # Check if any file NOT IN Manifest but IN binary package
+ if len(unverified_files) != 0:
+ raise DigestException(
+ f"Unknown files exists: {str(unverified_files)} in {self.gpkg_file}"
+ )
+
+ # Save current Manifest for other operations.
+ self.manifest_old = manifest.copy()
+
+ def _generate_metadata_from_dir(self, metadata_dir):
+ """
+ read all files in metadata_dir and return as dict
+ """
+ metadata = {}
+ metadata_dir = normalize_path(
+ _unicode_decode(metadata_dir, encoding=_encodings["fs"], errors="strict")
+ )
+ for parent, dirs, files in os.walk(metadata_dir):
+ for f in files:
+ try:
+ f = _unicode_decode(f, encoding=_encodings["fs"], errors="strict")
+ except UnicodeDecodeError:
+ continue
+ with open(os.path.join(parent, f), "rb") as metafile:
+ metadata[f] = metafile.read()
+ return metadata
+
+ def _get_binary_cmd(self, compression, mode):
+ """
+ get command list form portage and try match compressor
+ """
+ if compression not in _compressors:
+ raise InvalidCompressionMethod(compression)
+
+ compressor = _compressors[compression]
+ if mode not in compressor:
+ raise InvalidCompressionMethod("{}: {}".format(compression, mode))
+
+ cmd = shlex_split(varexpand(compressor[mode], mydict=self.settings))
+ # Filter empty elements that make Popen fail
+ cmd = [x for x in cmd if x != ""]
+
+ if (not cmd) and ((mode + "_alt") in compressor):
+ cmd = shlex_split(
+ varexpand(compressor[mode + "_alt"], mydict=self.settings)
+ )
+ cmd = [x for x in cmd if x != ""]
+
+ if not cmd:
+ raise CompressorNotFound(compression)
+ if not find_binary(cmd[0]):
+ raise CompressorNotFound(cmd[0])
+
+ return cmd
+
+ def _get_compression_cmd(self, compression=None):
+ """
+ return compression command for Popen
+ """
+ if compression is None:
+ compression = self.compression
+ if compression is None:
+ return None
+ else:
+ return self._get_binary_cmd(compression, "compress")
+
+ def _get_decompression_cmd(self, compression=None):
+ """
+ return decompression command for Popen
+ """
+ if compression is None:
+ compression = self.compression
+ if compression is None:
+ return None
+ else:
+ return self._get_binary_cmd(compression, "decompress")
+
+ def _get_tar_format(self, fileobj):
+ """
+ Try to detect tar version
+ """
+ old_position = fileobj.tell()
+ fileobj.seek(0x101)
+ magic = fileobj.read(8)
+ fileobj.seek(0x9C)
+ typeflag = fileobj.read(1)
+ fileobj.seek(old_position)
+
+ if magic == b"ustar \x00":
+ return tarfile.GNU_FORMAT
+ elif magic == b"ustar\x0000":
+ if typeflag == b"x" or typeflag == b"g":
+ return tarfile.PAX_FORMAT
+ else:
+ return tarfile.USTAR_FORMAT
+
+ return None
+
+ def _get_tar_format_from_stats(
+ self,
+ image_max_prefix_length,
+ image_max_name_length,
+ image_max_linkname_length,
+ image_max_file_size,
+ image_total_size,
+ ):
+ """
+ Choose the corresponding tar format according to
+ the image information
+ """
+ # Max possible size in UStar is 8 GiB (8589934591 bytes)
+ # stored in 11 octets
+ # Use 8000000000, just in case we need add something extra
+
+ # Total size > 8 GiB, container need use GNU tar format
+ if image_total_size < 8000000000:
+ container_tar_format = tarfile.USTAR_FORMAT
+ else:
+ container_tar_format = tarfile.GNU_FORMAT
+
+ # Image at least one file > 8 GiB, image need use GNU tar format
+ if image_max_file_size < 8000000000:
+ image_tar_format = tarfile.USTAR_FORMAT
+ else:
+ image_tar_format = tarfile.GNU_FORMAT
+
+ # UStar support max 155 prefix length, 100 file name and 100 link name,
+ # ends with \x00. If any exceeded, failback to GNU format.
+ if image_max_prefix_length >= 155:
+ image_tar_format = tarfile.GNU_FORMAT
+
+ if image_max_name_length >= 100:
+ image_tar_format = tarfile.GNU_FORMAT
+
+ if image_max_linkname_length >= 100:
+ image_tar_format = tarfile.GNU_FORMAT
+ return container_tar_format, image_tar_format
+
+ def _check_pre_image_files(self, root_dir, image_prefix="image"):
+ """
+ Check the pre image files size and path, return the longest
+ path length, largest single file size, and total files size.
+ """
+ image_prefix_length = len(image_prefix) + 1
+ root_dir = os.path.join(
+ normalize_path(
+ _unicode_decode(root_dir, encoding=_encodings["fs"], errors="strict")
+ ),
+ "",
+ )
+ root_dir_length = len(
+ _unicode_encode(root_dir, encoding=_encodings["fs"], errors="strict")
+ )
+
+ image_max_prefix_length = 0
+ image_max_name_length = 0
+ image_max_link_length = 0
+ image_max_file_size = 0
+ image_total_size = 0
+
+ for parent, dirs, files in os.walk(root_dir):
+ parent = _unicode_decode(parent, encoding=_encodings["fs"], errors="strict")
+ for d in dirs:
+ try:
+ d = _unicode_decode(d, encoding=_encodings["fs"], errors="strict")
+ except UnicodeDecodeError as err:
+ writemsg(colorize("BAD", "\n*** %s\n\n" % err), noiselevel=-1)
+ raise
+
+ d = os.path.join(parent, d)
+ prefix_length = (
+ len(_unicode_encode(d, encoding=_encodings["fs"], errors="strict"))
+ - root_dir_length
+ + image_prefix_length
+ )
+
+ if os.path.islink(d):
+ path_link = os.readlink(d)
+ path_link_length = len(
+ _unicode_encode(
+ path_link, encoding=_encodings["fs"], errors="strict"
+ )
+ )
+ image_max_link_length = max(image_max_link_length, path_link_length)
+
+ image_max_prefix_length = max(image_max_prefix_length, prefix_length)
+
+ for f in files:
+ try:
+ f = _unicode_decode(f, encoding=_encodings["fs"], errors="strict")
+ except UnicodeDecodeError as err:
+ writemsg(colorize("BAD", "\n*** %s\n\n" % err), noiselevel=-1)
+ raise
+
+ filename_length = len(
+ _unicode_encode(f, encoding=_encodings["fs"], errors="strict")
+ )
+ image_max_name_length = max(image_max_name_length, filename_length)
+
+ f = os.path.join(parent, f)
+ path_length = (
+ len(_unicode_encode(f, encoding=_encodings["fs"], errors="strict"))
+ - root_dir_length
+ + image_prefix_length
+ )
+
+ file_stat = os.lstat(f)
+
+ if os.path.islink(f):
+ path_link = os.readlink(f)
+ path_link_length = len(
+ _unicode_encode(
+ path_link, encoding=_encodings["fs"], errors="strict"
+ )
+ )
+ elif file_stat.st_nlink > 1:
+ # Hardlink exists
+ path_link_length = path_length
+ else:
+ path_link_length = 0
+
+ image_max_link_length = max(image_max_link_length, path_link_length)
+
+ try:
+ file_size = os.path.getsize(f)
+ except FileNotFoundError:
+ # Ignore file not found if symlink to non-existing file
+ if os.path.islink(f):
+ continue
+ else:
+ raise
+ image_total_size += file_size
+ image_max_file_size = max(image_max_file_size, file_size)
+
+ return (
+ image_max_prefix_length,
+ image_max_name_length,
+ image_max_link_length,
+ image_max_file_size,
+ image_total_size,
+ )
+
+ def _check_pre_quickpkg_files(self, contents, root):
+ """
+ Check the pre quickpkg files size and path, return the longest
+ path length, largest single file size, and total files size.
+ """
+ root_dir = os.path.join(
+ normalize_path(
+ _unicode_decode(root, encoding=_encodings["fs"], errors="strict")
+ ),
+ "",
+ )
+ root_dir_length = len(
+ _unicode_encode(root_dir, encoding=_encodings["fs"], errors="strict")
+ )
+
+ image_max_prefix_length = 0
+ image_max_name_length = 0
+ image_max_link_length = 0
+ image_max_file_size = 0
+ image_total_size = 0
+
+ paths = list(contents)
+ for path in paths:
+ try:
+ path = _unicode_decode(path, encoding=_encodings["fs"], errors="strict")
+ except UnicodeDecodeError as err:
+ writemsg(colorize("BAD", "\n*** %s\n\n" % err), noiselevel=-1)
+ raise
+
+ d, f = os.path.split(path)
+
+ prefix_length = (
+ len(_unicode_encode(d, encoding=_encodings["fs"], errors="strict"))
+ - root_dir_length
+ )
+ image_max_prefix_length = max(image_max_prefix_length, prefix_length)
+
+ filename_length = len(
+ _unicode_encode(f, encoding=_encodings["fs"], errors="strict")
+ )
+ image_max_name_length = max(image_max_name_length, filename_length)
+
+ path_length = (
+ len(_unicode_encode(path, encoding=_encodings["fs"], errors="strict"))
+ - root_dir_length
+ )
+
+ file_stat = os.lstat(path)
+
+ if os.path.islink(path):
+ path_link = os.readlink(path)
+ path_link_length = len(
+ _unicode_encode(
+ path_link, encoding=_encodings["fs"], errors="strict"
+ )
+ )
+ elif file_stat.st_nlink > 1:
+ # Hardlink exists
+ path_link_length = path_length
+ else:
+ path_link_length = 0
+
+ image_max_link_length = max(image_max_link_length, path_link_length)
+
+ if os.path.isfile(path):
+ try:
+ file_size = os.path.getsize(path)
+ except FileNotFoundError:
+ # Ignore file not found if symlink to non-existing file
+ if os.path.islink(path):
+ continue
+ else:
+ raise
+ image_total_size += file_size
+ if file_size > image_max_file_size:
+ image_max_file_size = file_size
+
+ return (
+ image_max_prefix_length,
+ image_max_name_length,
+ image_max_link_length,
+ image_max_file_size,
+ image_total_size,
+ )
+
+ def _create_tarinfo(self, file_name):
+ """
+ Create new tarinfo for the new file
+ """
+ if self.compression is None:
+ ext = ""
+ elif self.compression in self.ext_list:
+ ext = self.ext_list[self.compression]
+ else:
+ raise InvalidCompressionMethod(self.compression)
+
+ data_tarinfo = tarfile.TarInfo(
+ os.path.join(self.base_name, file_name + ".tar" + ext)
+ )
+ return data_tarinfo
+
+ def _extract_filename_compression(self, file_name):
+ """
+ Extract the file basename and compression method
+ """
+ file_name = os.path.basename(file_name)
+ if file_name.endswith(".tar"):
+ return file_name[:-4], None
+
+ for compression in self.ext_list:
+ if file_name.endswith(".tar" + self.ext_list[compression]):
+ return (
+ file_name[: -len(".tar" + self.ext_list[compression])],
+ compression,
+ )
+
+ raise InvalidCompressionMethod(file_name)
+
+ def _get_inner_tarinfo(self, tar, file_name):
+ """
+ Get inner tarinfo from given container.
+ Will try get file_name from correct basename first,
+ if it fail, try any file that have same name as file_name, and
+ return the first one.
+ """
+ if self.gpkg_version not in tar.getnames():
+ raise InvalidBinaryPackageFormat("Invalid gpkg file.")
+
+ # Try get file with correct basename
+ inner_tarinfo = None
+ if self.base_name is None:
+ base_name = ""
+ else:
+ base_name = self.base_name
+ all_files = tar.getmembers()
+ for f in all_files:
+ if os.path.dirname(f.name) == base_name:
+ try:
+ f_name, f_comp = self._extract_filename_compression(f.name)
+ except InvalidCompressionMethod:
+ continue
+
+ if f_name == file_name:
+ return f, f_comp
+
+ # If failed, try get any file name matched
+ if inner_tarinfo is None:
+ for f in all_files:
+ try:
+ f_name, f_comp = self._extract_filename_compression(f.name)
+ except InvalidCompressionMethod:
+ continue
+ if f_name == file_name:
+ if self.base_name is not None:
+ writemsg(
+ colorize(
+ "WARN", "Package basename mismatched, using " + f.name
+ )
+ )
+ self.base_name_alt = os.path.dirname(f.name)
+ return f, f_comp
+
+ # Not found
+ raise FileNotFound(f"File Not found: {file_name}")
diff --git a/lib/portage/output.py b/lib/portage/output.py
index 42f487f8a..77375b012 100644
--- a/lib/portage/output.py
+++ b/lib/portage/output.py
@@ -157,7 +157,7 @@ _styles["UNMERGE_WARN"] = ("red",)
_styles["SECURITY_WARN"] = ("red",)
_styles["MERGE_LIST_PROGRESS"] = ("yellow",)
_styles["PKG_BLOCKER"] = ("red",)
-_styles["PKG_BLOCKER_SATISFIED"] = ("darkblue",)
+_styles["PKG_BLOCKER_SATISFIED"] = ("teal",)
_styles["PKG_MERGE"] = ("darkgreen",)
_styles["PKG_MERGE_SYSTEM"] = ("darkgreen",)
_styles["PKG_MERGE_WORLD"] = ("green",)
@@ -165,8 +165,8 @@ _styles["PKG_BINARY_MERGE"] = ("purple",)
_styles["PKG_BINARY_MERGE_SYSTEM"] = ("purple",)
_styles["PKG_BINARY_MERGE_WORLD"] = ("fuchsia",)
_styles["PKG_UNINSTALL"] = ("red",)
-_styles["PKG_NOMERGE"] = ("darkblue",)
-_styles["PKG_NOMERGE_SYSTEM"] = ("darkblue",)
+_styles["PKG_NOMERGE"] = ("teal",)
+_styles["PKG_NOMERGE_SYSTEM"] = ("teal",)
_styles["PKG_NOMERGE_WORLD"] = ("blue",)
_styles["PROMPT_CHOICE_DEFAULT"] = ("green",)
_styles["PROMPT_CHOICE_OTHER"] = ("red",)
diff --git a/lib/portage/package/ebuild/_config/special_env_vars.py b/lib/portage/package/ebuild/_config/special_env_vars.py
index 06ae3aa39..9b2d77aea 100644
--- a/lib/portage/package/ebuild/_config/special_env_vars.py
+++ b/lib/portage/package/ebuild/_config/special_env_vars.py
@@ -86,6 +86,7 @@ environ_whitelist += [
"ACCEPT_LICENSE",
"BASH_ENV",
"BASH_FUNC____in_portage_iuse%%",
+ "BINPKG_FORMAT",
"BROOT",
"BUILD_PREFIX",
"COLUMNS",
diff --git a/lib/portage/package/ebuild/config.py b/lib/portage/package/ebuild/config.py
index b4d6862a3..8fe51784e 100644
--- a/lib/portage/package/ebuild/config.py
+++ b/lib/portage/package/ebuild/config.py
@@ -41,6 +41,7 @@ from portage.const import (
PORTAGE_BASE_PATH,
PRIVATE_PATH,
PROFILE_PATH,
+ SUPPORTED_GENTOO_BINPKG_FORMATS,
USER_CONFIG_PATH,
USER_VIRTUALS_FILE,
)
@@ -1513,6 +1514,15 @@ class config:
noiselevel=-1,
)
+ binpkg_format = self.get("BINPKG_FORMAT")
+ if binpkg_format:
+ if binpkg_format not in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ writemsg(
+ "!!! BINPKG_FORMAT contains invalid or "
+ "unsupported format: %s" % binpkg_fotmat,
+ noiselevel=-1,
+ )
+
binpkg_compression = self.get("BINPKG_COMPRESS")
if binpkg_compression:
try:
diff --git a/lib/portage/package/ebuild/doebuild.py b/lib/portage/package/ebuild/doebuild.py
index 9650a8444..eb719b1ea 100644
--- a/lib/portage/package/ebuild/doebuild.py
+++ b/lib/portage/package/ebuild/doebuild.py
@@ -66,6 +66,7 @@ from portage.const import (
INVALID_ENV_FILE,
MISC_SH_BINARY,
PORTAGE_PYM_PACKAGES,
+ SUPPORTED_GENTOO_BINPKG_FORMATS,
)
from portage.data import portage_gid, portage_uid, secpass, uid, userpriv_groups
from portage.dbapi.porttree import _parse_uri_map
@@ -118,6 +119,7 @@ from portage.util.futures import asyncio
from portage.util.futures.executor.fork import ForkExecutor
from portage.util.path import first_existing
from portage.util.socks5 import get_socks5_proxy
+from portage.util._dyn_libs.dyn_libs import check_dyn_libs_inconsistent
from portage.versions import _pkgsplit
from _emerge.BinpkgEnvExtractor import BinpkgEnvExtractor
from _emerge.EbuildBuildDir import EbuildBuildDir
@@ -126,7 +128,6 @@ from _emerge.EbuildSpawnProcess import EbuildSpawnProcess
from _emerge.Package import Package
from _emerge.RootConfig import RootConfig
-
_unsandboxed_phases = frozenset(
[
"clean",
@@ -649,6 +650,18 @@ def doebuild_environment(
mysettings["KV"] = ""
mysettings.backup_changes("KV")
+ binpkg_format = mysettings.get(
+ "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
+ )
+ if binpkg_format not in portage.const.SUPPORTED_GENTOO_BINPKG_FORMATS:
+ writemsg(
+ "!!! BINPKG_FORMAT contains invalid or "
+ "unsupported format: %s" % binpkg_fotmat,
+ noiselevel=-1,
+ )
+ binpkg_format = "xpak"
+ mysettings["BINPKG_FORMAT"] = binpkg_format
+
binpkg_compression = mysettings.get("BINPKG_COMPRESS", "bzip2")
try:
compression = _compressors[binpkg_compression]
@@ -2359,7 +2372,10 @@ def _check_build_log(mysettings, out=None):
setuptools_warn = set()
setuptools_warn_re = re.compile(r".*\/setuptools\/.*: .*Warning: (.*)")
# skip useless version normalization warnings
- setuptools_warn_ignore_re = [re.compile(r"Normalizing .*")]
+ setuptools_warn_ignore_re = [
+ re.compile(r"Normalizing .*"),
+ re.compile(r"setup.py install is deprecated"),
+ ]
def _eerror(lines):
for line in lines:
@@ -3091,6 +3107,14 @@ def _post_src_install_soname_symlinks(mysettings, out):
errors="strict",
) as f:
f.write(soname_deps.provides)
+ else:
+ if check_dyn_libs_inconsistent(image_dir, soname_deps.provides):
+ eerror(
+ "Error! Installing dynamic libraries (.so) with blank PROVIDES!",
+ phase="install",
+ key=mysettings.mycpv,
+ out=out,
+ )
if unrecognized_elf_files:
qa_msg = ["QA Notice: Unrecognized ELF file(s):"]
diff --git a/lib/portage/package/ebuild/fetch.py b/lib/portage/package/ebuild/fetch.py
index 8c64362c2..2d3625800 100644
--- a/lib/portage/package/ebuild/fetch.py
+++ b/lib/portage/package/ebuild/fetch.py
@@ -1485,10 +1485,11 @@ def fetch(
except EnvironmentError:
pass
elif not orig_digests:
- # We don't have a digest, but the file exists. We must
- # assume that it is fully downloaded.
+ # We don't have a digest, and the temporary file exists.
if not force:
- continue
+ # Try to resume this download when full
+ # download has not been explicitly forced.
+ fetched = 1
else:
if (
mydigests[myfile].get("size") is not None
diff --git a/lib/portage/package/ebuild/prepare_build_dirs.py b/lib/portage/package/ebuild/prepare_build_dirs.py
index 659198905..32a770c99 100644
--- a/lib/portage/package/ebuild/prepare_build_dirs.py
+++ b/lib/portage/package/ebuild/prepare_build_dirs.py
@@ -102,6 +102,15 @@ def prepare_build_dirs(myroot=None, settings=None, cleanup=False):
apply_secpass_permissions(
mysettings[dir_key], uid=portage_uid, gid=portage_gid
)
+ # The setgid bit prevents a lockfile group permission race for bug #468990.
+ ipc_kwargs = {}
+ if portage.data.secpass >= 1:
+ ipc_kwargs["gid"] = portage_gid
+ ipc_kwargs["mode"] = 0o2770
+ ensure_dirs(
+ os.path.join(mysettings["PORTAGE_BUILDDIR"], ".ipc"),
+ **ipc_kwargs,
+ )
except PermissionDenied as e:
writemsg(_("Permission Denied: %s\n") % str(e), noiselevel=-1)
return 1
diff --git a/lib/portage/process.py b/lib/portage/process.py
index e1bd2314e..84e09f8ec 100644
--- a/lib/portage/process.py
+++ b/lib/portage/process.py
@@ -84,7 +84,6 @@ if _fd_dir is not None:
raise
return range(max_fd_limit)
-
elif os.path.isdir("/proc/%s/fd" % portage.getpid()):
# In order for this function to work in forked subprocesses,
# os.getpid() must be called from inside the function.
@@ -95,7 +94,6 @@ elif os.path.isdir("/proc/%s/fd" % portage.getpid()):
if fd.isdigit()
)
-
else:
def get_open_fds():
diff --git a/lib/portage/tests/.gnupg/openpgp-revocs.d/06B3A311BD775C280D22A9305D90EA06352177F6.rev b/lib/portage/tests/.gnupg/openpgp-revocs.d/06B3A311BD775C280D22A9305D90EA06352177F6.rev
new file mode 100644
index 000000000..a6752fd30
--- /dev/null
+++ b/lib/portage/tests/.gnupg/openpgp-revocs.d/06B3A311BD775C280D22A9305D90EA06352177F6.rev
@@ -0,0 +1,37 @@
+This is a revocation certificate for the OpenPGP key:
+
+pub rsa4096 2020-07-14 [S]
+ 06B3A311BD775C280D22A9305D90EA06352177F6
+uid Gentoo Portage Test Trusted Key (Test Only, Do NOT Trust!!!) (Gentoo Test Key) <test@example.org>
+
+A revocation certificate is a kind of "kill switch" to publicly
+declare that a key shall not anymore be used. It is not possible
+to retract such a revocation certificate once it has been published.
+
+Use it to revoke this key in case of a compromise or loss of
+the secret key. However, if the secret key is still accessible,
+it is better to generate a new revocation certificate and give
+a reason for the revocation. For details see the description of
+of the gpg command "--generate-revocation" in the GnuPG manual.
+
+To avoid an accidental use of this file, a colon has been inserted
+before the 5 dashes below. Remove this colon with a text editor
+before importing and publishing this revocation certificate.
+
+:-----BEGIN PGP PUBLIC KEY BLOCK-----
+Comment: This is a revocation certificate
+
+iQI2BCABCAAgFiEEBrOjEb13XCgNIqkwXZDqBjUhd/YFAl8OFTwCHQAACgkQXZDq
+BjUhd/aXCA/+OgzosMDaDe5DNwkSi2yKdC2X18v8JcaYnXBUR93nXA0LVN7iVWkR
+WEH3NuVspQZ5vK+3AHTKabqZFC/buA5oQOH01Ncd4lQISfOOhFiBn5DIPX31BVT0
+iPmVkcxHAD4031ptP4oat6EFclT13SRchtlnAO04JofeHnzQIw3SozQGzXpAA1g4
+BogQ0HWA88HzuEYYE+e/yzZL4D496X1DTaXksg0Py5c4SS6u5pND6lcUtAGxAwa9
+sJFPs+coeURaRV99CrJfdh4u2OkvINTfrKOS6NFBQq6HVH5mLsRXZlcE4Oo4d+fN
+XoPrTZnRUqpJADUdjHFvO/lr0fArJTS5IQCVBNFeCMlvgmUPeKWJ1r6Uiwe/UHor
+9OP/tK97EqpsaXmHbo0jOUkn5iiUwy784+JBSSu/Q2NxqcBr74aaRdfxvs62dmv7
+droCDQi3ebqTdnlDSaeCIWHyVlSroOhZ+ZETVy193K1X7VXFX3hYKiJ3G8QZwy3e
+AlsVGjIHWfC+K+enIn+uwSUvOWPN3upK8kqMRuXvAOppFCE4sTqNbxUnHHXaqo/r
+s1q6zVsWVILBk97BHlJph2IaqhV7iIgPU97/r4U/BT11VqDFdVSHcXcs4PDNs5vh
+6qttaDiyDqZjwMr+0iDoouHxFpqY8e+3M2gycUgGr2XV6ML0pXE6BqA=
+=nIjC
+-----END PGP PUBLIC KEY BLOCK-----
diff --git a/lib/portage/tests/.gnupg/openpgp-revocs.d/8DEDA2CDED49C8809287B89D8812797DDF1DD192.rev b/lib/portage/tests/.gnupg/openpgp-revocs.d/8DEDA2CDED49C8809287B89D8812797DDF1DD192.rev
new file mode 100644
index 000000000..456e0aa50
--- /dev/null
+++ b/lib/portage/tests/.gnupg/openpgp-revocs.d/8DEDA2CDED49C8809287B89D8812797DDF1DD192.rev
@@ -0,0 +1,37 @@
+This is a revocation certificate for the OpenPGP key:
+
+pub rsa4096 2020-07-14 [S]
+ 8DEDA2CDED49C8809287B89D8812797DDF1DD192
+uid Gentoo Portage Test Untrusted Key (Test Only, Do NOT Trust!!!) (Gentoo Test Key) <test@example.org>
+
+A revocation certificate is a kind of "kill switch" to publicly
+declare that a key shall not anymore be used. It is not possible
+to retract such a revocation certificate once it has been published.
+
+Use it to revoke this key in case of a compromise or loss of
+the secret key. However, if the secret key is still accessible,
+it is better to generate a new revocation certificate and give
+a reason for the revocation. For details see the description of
+of the gpg command "--generate-revocation" in the GnuPG manual.
+
+To avoid an accidental use of this file, a colon has been inserted
+before the 5 dashes below. Remove this colon with a text editor
+before importing and publishing this revocation certificate.
+
+:-----BEGIN PGP PUBLIC KEY BLOCK-----
+Comment: This is a revocation certificate
+
+iQI2BCABCAAgFiEEje2ize1JyICSh7idiBJ5fd8d0ZIFAl8OFXUCHQAACgkQiBJ5
+fd8d0ZKdwxAAhmkC0V+OLyOU9PCV6ogD9/3b3nVqNIreoc+gxHTLmEvxiMSItqmq
+DkcW9RJKAduA/HiLZQ8Yzxw+ldC6kuWqYEjNpSM54VDkrgOePi8W1bVDTCoSp7bo
+0JOG4frieqIxA6lhAA2UppH7EPRXoODPLYqooNxWAs3xxVrR6eGAb5l8NXzrymvN
+acFfOZ0s5FgADQskQHWVq6TaJn9DrcZxd+b+plSwPYDXqzTChKQ5jw7uMAPUvDkG
+JUWgoKiKSrK64bslUq8aEDEZQ4uxjyEi6G0vO/wPL/ysGhS7KkPgCZsEfNjWjajb
+jAsdvl1raoHxK/O7llMNr9uRAZtC56pJ//SRDc3kylZrkAo0RNoXQFowT739HWei
+2UkCFDfz488VKKrOI8TzTyUvLFEo14ZAXGg1wdHaGnbYMzxpKjP15alOFo6fKIcS
+Kz1f/Mab4wf4Sg0XAjQ9pnai1/U9ZF3/NSnRtYgJkLCrIEtRLrgSHJsLDPxjCfGV
+jWszAbIk167aA0yKsSmuwkpc5bZqqBaTo904r857fxyt5Les6SOHsV7iNXt7F+am
+03Y6u6m2eROba7M67l115vTyYcw5EZVp5j0nI81PXsC9X2DD1ci5xrNmPyEeupC4
+7y7mcGbUYPJAJHJ0kHG4ZYLnNMl42ZYr1ssEeasDwUsLWgVqvx9RkKI=
+=kVUQ
+-----END PGP PUBLIC KEY BLOCK-----
diff --git a/lib/portage/tests/.gnupg/private-keys-v1.d/273B030399E7BEA66A9AD42216DE7CA17BA5D42E.key b/lib/portage/tests/.gnupg/private-keys-v1.d/273B030399E7BEA66A9AD42216DE7CA17BA5D42E.key
new file mode 100644
index 000000000..0bd1026ad
--- /dev/null
+++ b/lib/portage/tests/.gnupg/private-keys-v1.d/273B030399E7BEA66A9AD42216DE7CA17BA5D42E.key
Binary files differ
diff --git a/lib/portage/tests/.gnupg/private-keys-v1.d/C99796FB85B0C3DF03314A11B5850C51167D6282.key b/lib/portage/tests/.gnupg/private-keys-v1.d/C99796FB85B0C3DF03314A11B5850C51167D6282.key
new file mode 100644
index 000000000..8e29ef43c
--- /dev/null
+++ b/lib/portage/tests/.gnupg/private-keys-v1.d/C99796FB85B0C3DF03314A11B5850C51167D6282.key
Binary files differ
diff --git a/lib/portage/tests/.gnupg/pubring.kbx b/lib/portage/tests/.gnupg/pubring.kbx
new file mode 100644
index 000000000..f6367f83b
--- /dev/null
+++ b/lib/portage/tests/.gnupg/pubring.kbx
Binary files differ
diff --git a/lib/portage/tests/.gnupg/trustdb.gpg b/lib/portage/tests/.gnupg/trustdb.gpg
new file mode 100644
index 000000000..db5b1023b
--- /dev/null
+++ b/lib/portage/tests/.gnupg/trustdb.gpg
Binary files differ
diff --git a/lib/portage/tests/__init__.py b/lib/portage/tests/__init__.py
index 02d9c4932..f74f992d7 100644
--- a/lib/portage/tests/__init__.py
+++ b/lib/portage/tests/__init__.py
@@ -14,6 +14,7 @@ import portage
from portage import os
from portage import _encodings
from portage import _unicode_decode
+from portage.output import colorize
from portage.proxy.objectproxy import ObjectProxy
@@ -184,19 +185,43 @@ class TextTestResult(_TextTestResult):
self.todoed = []
self.portage_skipped = []
+ def addSuccess(self, test):
+ super(_TextTestResult, self).addSuccess(test)
+ if self.showAll:
+ self.stream.writeln(colorize("GOOD", "ok"))
+ elif self.dots:
+ self.stream.write(colorize("GOOD", "."))
+ self.stream.flush()
+
+ def addError(self, test, err):
+ super(_TextTestResult, self).addError(test, err)
+ if self.showAll:
+ self.stream.writeln(colorize("BAD", "ERROR"))
+ elif self.dots:
+ self.stream.write(colorize("HILITE", "E"))
+ self.stream.flush()
+
+ def addFailure(self, test, err):
+ super(_TextTestResult, self).addFailure(test, err)
+ if self.showAll:
+ self.stream.writeln(colorize("BAD", "FAIL"))
+ elif self.dots:
+ self.stream.write(colorize("BAD", "F"))
+ self.stream.flush()
+
def addTodo(self, test, info):
self.todoed.append((test, info))
if self.showAll:
- self.stream.writeln("TODO")
+ self.stream.writeln(colorize("BRACKET", "TODO"))
elif self.dots:
- self.stream.write(".")
+ self.stream.write(colorize("BRACKET", "."))
def addPortageSkip(self, test, info):
self.portage_skipped.append((test, info))
if self.showAll:
- self.stream.writeln("SKIP")
+ self.stream.writeln(colorize("WARN", "SKIP"))
elif self.dots:
- self.stream.write(".")
+ self.stream.write(colorize("WARN", "."))
def printErrors(self):
if self.dots or self.showAll:
@@ -331,7 +356,7 @@ class TextTestRunner(unittest.TextTestRunner):
)
self.stream.writeln()
if not result.wasSuccessful():
- self.stream.write("FAILED (")
+ self.stream.write(colorize("BAD", "FAILED") + " (")
failed = len(result.failures)
errored = len(result.errors)
if failed:
@@ -342,7 +367,7 @@ class TextTestRunner(unittest.TextTestRunner):
self.stream.write("errors=%d" % errored)
self.stream.writeln(")")
else:
- self.stream.writeln("OK")
+ self.stream.writeln(colorize("GOOD", "OK"))
return result
diff --git a/lib/portage/tests/ebuild/test_doebuild_spawn.py b/lib/portage/tests/ebuild/test_doebuild_spawn.py
index ef0ae5847..d142ec41e 100644
--- a/lib/portage/tests/ebuild/test_doebuild_spawn.py
+++ b/lib/portage/tests/ebuild/test_doebuild_spawn.py
@@ -81,6 +81,7 @@ class DoebuildSpawnTestCase(TestCase):
settings["T"] = os.path.join(settings["PORTAGE_BUILDDIR"], "temp")
for x in ("PORTAGE_BUILDDIR", "HOME", "T"):
os.makedirs(settings[x])
+ os.makedirs(os.path.join(settings["PORTAGE_BUILDDIR"], ".ipc"))
# Create a fake environment, to pretend as if the ebuild
# has been sourced already.
open(os.path.join(settings["T"], "environment"), "wb").close()
diff --git a/lib/portage/tests/ebuild/test_ipc_daemon.py b/lib/portage/tests/ebuild/test_ipc_daemon.py
index e20b6fff1..0ac3d3c32 100644
--- a/lib/portage/tests/ebuild/test_ipc_daemon.py
+++ b/lib/portage/tests/ebuild/test_ipc_daemon.py
@@ -66,10 +66,10 @@ class IpcDaemonTestCase(TestCase):
build_dir = EbuildBuildDir(scheduler=event_loop, settings=env)
event_loop.run_until_complete(build_dir.async_lock())
- ensure_dirs(env["PORTAGE_BUILDDIR"])
+ ensure_dirs(os.path.join(env["PORTAGE_BUILDDIR"], ".ipc"))
- input_fifo = os.path.join(env["PORTAGE_BUILDDIR"], ".ipc_in")
- output_fifo = os.path.join(env["PORTAGE_BUILDDIR"], ".ipc_out")
+ input_fifo = os.path.join(env["PORTAGE_BUILDDIR"], ".ipc", "in")
+ output_fifo = os.path.join(env["PORTAGE_BUILDDIR"], ".ipc", "out")
os.mkfifo(input_fifo)
os.mkfifo(output_fifo)
diff --git a/lib/portage/tests/emerge/test_simple.py b/lib/portage/tests/emerge/test_simple.py
index 3a8bf3764..6d0dae0dd 100644
--- a/lib/portage/tests/emerge/test_simple.py
+++ b/lib/portage/tests/emerge/test_simple.py
@@ -3,6 +3,7 @@
import argparse
import subprocess
+import sys
import portage
from portage import shutil, os
@@ -11,6 +12,7 @@ from portage.const import (
BINREPOS_CONF_FILE,
PORTAGE_PYM_PATH,
USER_CONFIG_PATH,
+ SUPPORTED_GENTOO_BINPKG_FORMATS,
)
from portage.cache.mappings import Mapping
from portage.process import find_binary
@@ -19,6 +21,7 @@ from portage.tests.resolver.ResolverPlayground import ResolverPlayground
from portage.tests.util.test_socks5 import AsyncHTTPServer
from portage.util import ensure_dirs, find_updated_config_files, shlex_split
from portage.util.futures import asyncio
+from portage.output import colorize
class BinhostContentMap(Mapping):
@@ -223,17 +226,28 @@ call_has_and_best_version() {
),
)
- playground = ResolverPlayground(
- ebuilds=ebuilds, installed=installed, debug=debug
- )
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ ebuilds=ebuilds,
+ installed=installed,
+ debug=debug,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
- loop = asyncio._wrap_loop()
- loop.run_until_complete(
- asyncio.ensure_future(
- self._async_test_simple(playground, metadata_xml_files, loop=loop),
- loop=loop,
- )
- )
+ loop = asyncio._wrap_loop()
+ loop.run_until_complete(
+ asyncio.ensure_future(
+ self._async_test_simple(
+ playground, metadata_xml_files, loop=loop
+ ),
+ loop=loop,
+ )
+ )
async def _async_test_simple(self, playground, metadata_xml_files, loop):
@@ -327,6 +341,15 @@ call_has_and_best_version() {
path=binhost_remote_path,
)
+ binpkg_format = settings.get(
+ "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
+ )
+ self.assertIn(binpkg_format, ("xpak", "gpkg"))
+ if binpkg_format == "xpak":
+ foo_filename = "foo-0-1.xpak"
+ elif binpkg_format == "gpkg":
+ foo_filename = "foo-0-1.gpkg.tar"
+
test_commands = ()
if hasattr(argparse.ArgumentParser, "parse_intermixed_args"):
@@ -387,13 +410,13 @@ call_has_and_best_version() {
rm_cmd + ("-rf", cachedir),
emerge_cmd + ("--oneshot", "virtual/foo"),
lambda: self.assertFalse(
- os.path.exists(os.path.join(pkgdir, "virtual", "foo", "foo-0-1.xpak"))
+ os.path.exists(os.path.join(pkgdir, "virtual", "foo", foo_filename))
),
({"FEATURES": "unmerge-backup"},)
+ emerge_cmd
+ ("--unmerge", "virtual/foo"),
lambda: self.assertTrue(
- os.path.exists(os.path.join(pkgdir, "virtual", "foo", "foo-0-1.xpak"))
+ os.path.exists(os.path.join(pkgdir, "virtual", "foo", foo_filename))
),
emerge_cmd + ("--pretend", "dev-libs/A"),
ebuild_cmd + (test_ebuild, "manifest", "clean", "package", "merge"),
diff --git a/lib/portage/tests/gpkg/__init__.py b/lib/portage/tests/gpkg/__init__.py
new file mode 100644
index 000000000..532918b6a
--- /dev/null
+++ b/lib/portage/tests/gpkg/__init__.py
@@ -0,0 +1,2 @@
+# Copyright 2011 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
diff --git a/lib/portage/tests/gpkg/__test__.py b/lib/portage/tests/gpkg/__test__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/lib/portage/tests/gpkg/__test__.py
diff --git a/lib/portage/tests/gpkg/test_gpkg_checksum.py b/lib/portage/tests/gpkg/test_gpkg_checksum.py
new file mode 100644
index 000000000..c78045c34
--- /dev/null
+++ b/lib/portage/tests/gpkg/test_gpkg_checksum.py
@@ -0,0 +1,396 @@
+# Copright Gentoo Foundation 2006-2020
+# Portage Unit Testing Functionality
+
+import io
+import sys
+import tarfile
+import tempfile
+from os import urandom
+
+from portage import os
+from portage import shutil
+from portage.tests import TestCase
+from portage.tests.resolver.ResolverPlayground import ResolverPlayground
+from portage.gpkg import gpkg
+from portage.exception import (
+ InvalidBinaryPackageFormat,
+ DigestException,
+ MissingSignature,
+)
+
+
+class test_gpkg_checksum_case(TestCase):
+ def test_gpkg_missing_header(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": (
+ 'FEATURES="${FEATURES} -binpkg-signing '
+ '-binpkg-request-signature -gpg-keepalive"',
+ ),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+
+ try:
+ settings = playground.settings
+ orig_full_path = os.path.join(tmpdir, "orig/")
+ os.makedirs(orig_full_path)
+
+ data = urandom(1048576)
+ with open(os.path.join(orig_full_path, "data"), "wb") as f:
+ f.write(data)
+
+ binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
+ binpkg_1.compress(orig_full_path, {})
+
+ with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
+ with tarfile.open(
+ os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
+ ) as tar_2:
+ for f in tar_1.getmembers():
+ if f.name != binpkg_1.gpkg_version:
+ tar_2.addfile(f, tar_1.extractfile(f))
+
+ binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
+
+ self.assertRaises(
+ InvalidBinaryPackageFormat,
+ binpkg_2.decompress,
+ os.path.join(tmpdir, "test"),
+ )
+ finally:
+ shutil.rmtree(tmpdir)
+ playground.cleanup()
+
+ def test_gpkg_missing_manifest(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": (
+ 'FEATURES="${FEATURES} -binpkg-signing '
+ '-binpkg-request-signature -gpg-keepalive"',
+ ),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+
+ try:
+ settings = playground.settings
+ orig_full_path = os.path.join(tmpdir, "orig/")
+ os.makedirs(orig_full_path)
+
+ data = urandom(1048576)
+ with open(os.path.join(orig_full_path, "data"), "wb") as f:
+ f.write(data)
+
+ binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
+ binpkg_1.compress(orig_full_path, {})
+
+ with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
+ with tarfile.open(
+ os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
+ ) as tar_2:
+ for f in tar_1.getmembers():
+ if f.name != "Manifest":
+ tar_2.addfile(f, tar_1.extractfile(f))
+
+ binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
+
+ self.assertRaises(
+ MissingSignature, binpkg_2.decompress, os.path.join(tmpdir, "test")
+ )
+ finally:
+ shutil.rmtree(tmpdir)
+ playground.cleanup()
+
+ def test_gpkg_missing_files(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": (
+ 'FEATURES="${FEATURES} -binpkg-signing '
+ '-binpkg-request-signature -gpg-keepalive"',
+ ),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+
+ try:
+ settings = playground.settings
+ orig_full_path = os.path.join(tmpdir, "orig/")
+ os.makedirs(orig_full_path)
+
+ data = urandom(1048576)
+ with open(os.path.join(orig_full_path, "data"), "wb") as f:
+ f.write(data)
+
+ data = urandom(1048576)
+ with open(os.path.join(orig_full_path, "data2"), "wb") as f:
+ f.write(data)
+
+ binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
+ binpkg_1.compress(orig_full_path, {})
+
+ with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
+ with tarfile.open(
+ os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
+ ) as tar_2:
+ for f in tar_1.getmembers():
+ if "image.tar" not in f.name:
+ tar_2.addfile(f, tar_1.extractfile(f))
+
+ binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
+
+ self.assertRaises(
+ DigestException, binpkg_2.decompress, os.path.join(tmpdir, "test")
+ )
+ finally:
+ shutil.rmtree(tmpdir)
+ playground.cleanup()
+
+ def test_gpkg_extra_files(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": (
+ 'FEATURES="${FEATURES} -binpkg-signing '
+ '-binpkg-request-signature -gpg-keepalive"',
+ ),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+
+ try:
+ settings = playground.settings
+ orig_full_path = os.path.join(tmpdir, "orig/")
+ os.makedirs(orig_full_path)
+
+ data = urandom(1048576)
+ with open(os.path.join(orig_full_path, "data"), "wb") as f:
+ f.write(data)
+
+ binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
+ binpkg_1.compress(orig_full_path, {})
+
+ with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
+ with tarfile.open(
+ os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
+ ) as tar_2:
+ for f in tar_1.getmembers():
+ tar_2.addfile(f, tar_1.extractfile(f))
+ data_tarinfo = tarfile.TarInfo("data2")
+ data_tarinfo.size = len(data)
+ data2 = io.BytesIO(data)
+ tar_2.addfile(data_tarinfo, data2)
+ data2.close()
+
+ binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
+
+ self.assertRaises(
+ DigestException, binpkg_2.decompress, os.path.join(tmpdir, "test")
+ )
+ finally:
+ shutil.rmtree(tmpdir)
+ playground.cleanup()
+
+ def test_gpkg_incorrect_checksum(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": (
+ 'FEATURES="${FEATURES} -binpkg-signing '
+ '-binpkg-request-signature -gpg-keepalive"',
+ ),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+
+ try:
+ settings = playground.settings
+ orig_full_path = os.path.join(tmpdir, "orig/")
+ os.makedirs(orig_full_path)
+
+ data = urandom(1048576)
+ with open(os.path.join(orig_full_path, "data"), "wb") as f:
+ f.write(data)
+
+ binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
+ binpkg_1.compress(orig_full_path, {})
+
+ with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
+ with tarfile.open(
+ os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
+ ) as tar_2:
+ for f in tar_1.getmembers():
+ if f.name == "Manifest":
+ data = io.BytesIO(tar_1.extractfile(f).read())
+ data_view = data.getbuffer()
+ data_view[-16:] = b"20a6d80ab0320fh9"
+ del data_view
+ tar_2.addfile(f, data)
+ data.close()
+ else:
+ tar_2.addfile(f, tar_1.extractfile(f))
+
+ binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
+
+ self.assertRaises(
+ DigestException, binpkg_2.decompress, os.path.join(tmpdir, "test")
+ )
+ finally:
+ shutil.rmtree(tmpdir)
+ playground.cleanup()
+
+ def test_gpkg_duplicate_files(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": (
+ 'FEATURES="${FEATURES} -binpkg-signing '
+ '-binpkg-request-signature -gpg-keepalive"',
+ ),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+
+ try:
+ settings = playground.settings
+ orig_full_path = os.path.join(tmpdir, "orig/")
+ os.makedirs(orig_full_path)
+
+ data = urandom(100)
+ with open(os.path.join(orig_full_path, "data"), "wb") as f:
+ f.write(data)
+
+ binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
+ binpkg_1.compress(orig_full_path, {})
+
+ with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
+ with tarfile.open(
+ os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
+ ) as tar_2:
+ for f in tar_1.getmembers():
+ tar_2.addfile(f, tar_1.extractfile(f))
+ tar_2.addfile(f, tar_1.extractfile(f))
+
+ binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
+
+ self.assertRaises(
+ InvalidBinaryPackageFormat,
+ binpkg_2.decompress,
+ os.path.join(tmpdir, "test"),
+ )
+ finally:
+ shutil.rmtree(tmpdir)
+ playground.cleanup()
+
+ def test_gpkg_manifest_duplicate_files(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": (
+ 'FEATURES="${FEATURES} -binpkg-signing '
+ '-binpkg-request-signature -gpg-keepalive"',
+ ),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+
+ try:
+ settings = playground.settings
+ orig_full_path = os.path.join(tmpdir, "orig/")
+ os.makedirs(orig_full_path)
+
+ data = urandom(100)
+ with open(os.path.join(orig_full_path, "data"), "wb") as f:
+ f.write(data)
+
+ binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
+ binpkg_1.compress(orig_full_path, {})
+
+ with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
+ with tarfile.open(
+ os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
+ ) as tar_2:
+ for f in tar_1.getmembers():
+ if f.name == "Manifest":
+ manifest = tar_1.extractfile(f).read()
+ data = io.BytesIO(manifest)
+ data.seek(io.SEEK_END)
+ data.write(b"\n")
+ data.write(manifest)
+ f.size = data.tell()
+ data.seek(0)
+ tar_2.addfile(f, data)
+ data.close()
+ else:
+ tar_2.addfile(f, tar_1.extractfile(f))
+
+ binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
+
+ self.assertRaises(
+ DigestException, binpkg_2.decompress, os.path.join(tmpdir, "test")
+ )
+ finally:
+ shutil.rmtree(tmpdir)
+ playground.cleanup()
+
+ def test_gpkg_different_size_file(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": (
+ 'FEATURES="${FEATURES} -binpkg-signing '
+ '-binpkg-request-signature -gpg-keepalive"',
+ ),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+
+ try:
+ settings = playground.settings
+ orig_full_path = os.path.join(tmpdir, "orig/")
+ os.makedirs(orig_full_path)
+
+ data = urandom(100)
+ with open(os.path.join(orig_full_path, "data"), "wb") as f:
+ f.write(data)
+
+ binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
+ binpkg_1.compress(orig_full_path, {})
+
+ with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
+ with tarfile.open(
+ os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
+ ) as tar_2:
+ for f in tar_1.getmembers():
+ tar_2.addfile(f, tar_1.extractfile(f))
+ tar_2.addfile(f, tar_1.extractfile(f))
+
+ binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
+
+ self.assertRaises(
+ InvalidBinaryPackageFormat,
+ binpkg_2.decompress,
+ os.path.join(tmpdir, "test"),
+ )
+ finally:
+ shutil.rmtree(tmpdir)
+ playground.cleanup()
diff --git a/lib/portage/tests/gpkg/test_gpkg_gpg.py b/lib/portage/tests/gpkg/test_gpkg_gpg.py
new file mode 100644
index 000000000..906608dff
--- /dev/null
+++ b/lib/portage/tests/gpkg/test_gpkg_gpg.py
@@ -0,0 +1,398 @@
+# Copright Gentoo Foundation 2006-2020
+# Portage Unit Testing Functionality
+
+import io
+import sys
+import tarfile
+import tempfile
+from os import urandom
+
+from portage import os
+from portage import shutil
+from portage.tests import TestCase
+from portage.tests.resolver.ResolverPlayground import ResolverPlayground
+from portage.gpkg import gpkg
+from portage.gpg import GPG
+from portage.exception import MissingSignature, InvalidSignature
+
+
+class test_gpkg_gpg_case(TestCase):
+ def test_gpkg_missing_manifest_signature(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": (
+ 'FEATURES="${FEATURES} binpkg-signing ' 'binpkg-request-signature"',
+ 'BINPKG_FORMAT="gpkg"',
+ ),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+
+ try:
+ settings = playground.settings
+ gpg = GPG(settings)
+ gpg.unlock()
+ orig_full_path = os.path.join(tmpdir, "orig/")
+ os.makedirs(orig_full_path)
+
+ data = urandom(1048576)
+ with open(os.path.join(orig_full_path, "data"), "wb") as f:
+ f.write(data)
+
+ binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
+ binpkg_1.compress(orig_full_path, {})
+
+ with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
+ with tarfile.open(
+ os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
+ ) as tar_2:
+ for f in tar_1.getmembers():
+ if f.name == "Manifest":
+ manifest = tar_1.extractfile(f).read().decode("UTF-8")
+ manifest = manifest.replace(
+ "-----BEGIN PGP SIGNATURE-----", ""
+ )
+ manifest = manifest.replace(
+ "-----END PGP SIGNATURE-----", ""
+ )
+ manifest_data = io.BytesIO(manifest.encode("UTF-8"))
+ manifest_data.seek(0, io.SEEK_END)
+ f.size = manifest_data.tell()
+ manifest_data.seek(0)
+ tar_2.addfile(f, manifest_data)
+ else:
+ tar_2.addfile(f, tar_1.extractfile(f))
+
+ binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
+
+ self.assertRaises(
+ InvalidSignature, binpkg_2.decompress, os.path.join(tmpdir, "test")
+ )
+ finally:
+ shutil.rmtree(tmpdir)
+ playground.cleanup()
+
+ def test_gpkg_missing_signature(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": (
+ 'FEATURES="${FEATURES} binpkg-signing ' 'binpkg-request-signature"',
+ 'BINPKG_FORMAT="gpkg"',
+ ),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+
+ try:
+ settings = playground.settings
+ gpg = GPG(settings)
+ gpg.unlock()
+ orig_full_path = os.path.join(tmpdir, "orig/")
+ os.makedirs(orig_full_path)
+
+ data = urandom(1048576)
+ with open(os.path.join(orig_full_path, "data"), "wb") as f:
+ f.write(data)
+
+ binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
+ binpkg_1.compress(orig_full_path, {})
+
+ with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
+ with tarfile.open(
+ os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
+ ) as tar_2:
+ for f in tar_1.getmembers():
+ if f.name.endswith(".sig"):
+ pass
+ else:
+ tar_2.addfile(f, tar_1.extractfile(f))
+
+ binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
+ self.assertRaises(
+ MissingSignature, binpkg_2.decompress, os.path.join(tmpdir, "test")
+ )
+
+ finally:
+ shutil.rmtree(tmpdir)
+ playground.cleanup()
+
+ def test_gpkg_ignore_signature(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": (
+ 'FEATURES="${FEATURES} binpkg-signing ' 'binpkg-ignore-signature"',
+ 'BINPKG_FORMAT="gpkg"',
+ ),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+
+ try:
+ settings = playground.settings
+ gpg = GPG(settings)
+ gpg.unlock()
+ orig_full_path = os.path.join(tmpdir, "orig/")
+ os.makedirs(orig_full_path)
+
+ data = urandom(1048576)
+ with open(os.path.join(orig_full_path, "data"), "wb") as f:
+ f.write(data)
+
+ binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
+ binpkg_1.compress(orig_full_path, {})
+
+ with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
+ with tarfile.open(
+ os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
+ ) as tar_2:
+ for f in tar_1.getmembers():
+ if f.name == "Manifest.sig":
+ pass
+ else:
+ tar_2.addfile(f, tar_1.extractfile(f))
+
+ binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
+ binpkg_2.decompress(os.path.join(tmpdir, "test"))
+ finally:
+ shutil.rmtree(tmpdir)
+ playground.cleanup()
+
+ def test_gpkg_auto_use_signature(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": (
+ 'FEATURES="${FEATURES} binpkg-signing '
+ '-binpkg-request-signature"',
+ 'BINPKG_FORMAT="gpkg"',
+ ),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+
+ try:
+ settings = playground.settings
+ gpg = GPG(settings)
+ gpg.unlock()
+ orig_full_path = os.path.join(tmpdir, "orig/")
+ os.makedirs(orig_full_path)
+
+ data = urandom(1048576)
+ with open(os.path.join(orig_full_path, "data"), "wb") as f:
+ f.write(data)
+
+ binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
+ binpkg_1.compress(orig_full_path, {})
+
+ with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
+ with tarfile.open(
+ os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
+ ) as tar_2:
+ for f in tar_1.getmembers():
+ if f.name.endswith(".sig"):
+ pass
+ else:
+ tar_2.addfile(f, tar_1.extractfile(f))
+
+ binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
+ self.assertRaises(
+ MissingSignature, binpkg_2.decompress, os.path.join(tmpdir, "test")
+ )
+ finally:
+ shutil.rmtree(tmpdir)
+ playground.cleanup()
+
+ def test_gpkg_invalid_signature(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": (
+ 'FEATURES="${FEATURES} binpkg-signing ' 'binpkg-request-signature"',
+ 'BINPKG_FORMAT="gpkg"',
+ ),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+
+ try:
+ settings = playground.settings
+ gpg = GPG(settings)
+ gpg.unlock()
+ orig_full_path = os.path.join(tmpdir, "orig/")
+ os.makedirs(orig_full_path)
+
+ data = urandom(1048576)
+ with open(os.path.join(orig_full_path, "data"), "wb") as f:
+ f.write(data)
+
+ binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
+ binpkg_1.compress(orig_full_path, {})
+
+ with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
+ with tarfile.open(
+ os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
+ ) as tar_2:
+ for f in tar_1.getmembers():
+ if f.name == "Manifest":
+ sig = b"""
+-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA512
+
+DATA test/image.tar.zst 1049649 BLAKE2B 3112adba9c09023962f26d9dcbf8e74107c05220f2f29aa2ce894f8a4104c3bb238f87095df73735befcf1e1f6039fc3abf4defa87e68ce80f33dd01e09c055a SHA512 9f584727f2e20a50a30e0077b94082c8c1f517ebfc9978eb3281887e24458108e73d1a2ce82eb0b59f5df7181597e4b0a297ae68bbfb36763aa052e6bdbf2c59
+DATA test/image.tar.zst.sig 833 BLAKE2B 214724ae4ff9198879c8c960fd8167632e27982c2278bb873f195abe75b75afa1ebed4c37ec696f5f5bc35c3a1184b60e0b50d56695b072b254f730db01eddb5 SHA512 67316187da8bb6b7a5f9dc6a42ed5c7d72c6184483a97f23c0bebd8b187ac9268e0409eb233c935101606768718c99eaa5699037d6a68c2d88c9ed5331a3f73c
+-----BEGIN PGP SIGNATURE-----
+
+iQIzBAEBCgAdFiEEBrOjEb13XCgNIqkwXZDqBjUhd/YFAmFazXEACgkQXZDqBjUh
+d/YFZA//eiXkYAS2NKxim6Ppr1HcZdjU1f6H+zyQzC7OdPkAh7wsVXpSr1aq+giD
+G4tNtI6nsFokpA5CMhDf+ffBofKmFY5plk9zyQHr43N/RS5G6pcb2LHk0mQqgIdB
+EsZRRD75Na4uGDWjuNHRmsasPTsc9qyW7FLckjwUsVmk9foAoiLYYaTsilsEGqXD
+Bl/Z6PaQXvdd8txbcP6dOXfhVT06b+RWcnHI06KQrmFkZjZQh/7bCIeCVwNbXr7d
+Obo8SVzCrQbTONei57AkyuRfnPqBfP61k8rQtcDUmCckQQfyaRwoW2nDIewOPfIH
+xfvM137to2GEI2RR1TpWmGfu3iQzgC71f4svdX9Tyi5N7aFmfud7LZs6/Un3IdVk
+ZH9/AmRzeH6hKllqSv/6WuhjsTNvr0bOzGbskkhqlLga2tml08gHFYOMWRJb/bRz
+N8FZMhHzFoc0hsG8SU9uC+OeW+y5NdqpbRnQwgABmAiKEpgAPnABTsr0HjyxvjY+
+uCUdvMMHvnTxTjNEZ3Q+UQ2VsSoZzPbW9Y4PuM0XxxmTI8htdn4uIhy9dLNPsJmB
+eTE8aov/1uKq9VMsYC8wcx5vLMaR7/O/9XstP+r6PaZwiLlyrKHGexV4O52sj6LC
+qGAN3VUF+8EsdcsV781H0F86PANhyBgEYTGDrnItTGe3/vAPjCo=
+=S/Vn
+-----END PGP SIGNATURE-----
+"""
+ data = io.BytesIO(sig)
+ f.size = len(sig)
+ tar_2.addfile(f, data)
+ data.close()
+ else:
+ tar_2.addfile(f, tar_1.extractfile(f))
+
+ binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
+ self.assertRaises(
+ InvalidSignature, binpkg_2.decompress, os.path.join(tmpdir, "test")
+ )
+ finally:
+ shutil.rmtree(tmpdir)
+ playground.cleanup()
+
+ def test_gpkg_untrusted_signature(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ gpg_test_path = os.environ["PORTAGE_GNUPGHOME"]
+
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": (
+ 'FEATURES="${FEATURES} binpkg-signing ' 'binpkg-request-signature"',
+ 'BINPKG_FORMAT="gpkg"',
+ f'BINPKG_GPG_SIGNING_BASE_COMMAND="flock {gpg_test_path}/portage-binpkg-gpg.lock /usr/bin/gpg --sign --armor --batch --no-tty --yes --pinentry-mode loopback --passphrase GentooTest [PORTAGE_CONFIG]"',
+ 'BINPKG_GPG_SIGNING_DIGEST="SHA512"',
+ f'BINPKG_GPG_SIGNING_GPG_HOME="{gpg_test_path}"',
+ 'BINPKG_GPG_SIGNING_KEY="0x8812797DDF1DD192"',
+ 'BINPKG_GPG_VERIFY_BASE_COMMAND="/usr/bin/gpg --verify --batch --no-tty --yes --no-auto-check-trustdb --status-fd 1 [PORTAGE_CONFIG] [SIGNATURE]"',
+ f'BINPKG_GPG_VERIFY_GPG_HOME="{gpg_test_path}"',
+ ),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+
+ try:
+ settings = playground.settings
+ gpg = GPG(settings)
+ gpg.unlock()
+ orig_full_path = os.path.join(tmpdir, "orig/")
+ os.makedirs(orig_full_path)
+
+ data = urandom(1048576)
+ with open(os.path.join(orig_full_path, "data"), "wb") as f:
+ f.write(data)
+
+ binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
+ binpkg_1.compress(orig_full_path, {})
+
+ binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
+ self.assertRaises(
+ InvalidSignature, binpkg_2.decompress, os.path.join(tmpdir, "test")
+ )
+
+ finally:
+ shutil.rmtree(tmpdir)
+ playground.cleanup()
+
+ def test_gpkg_unknown_signature(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": (
+ 'FEATURES="${FEATURES} binpkg-signing ' 'binpkg-request-signature"',
+ 'BINPKG_FORMAT="gpkg"',
+ ),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+
+ try:
+ settings = playground.settings
+ gpg = GPG(settings)
+ gpg.unlock()
+ orig_full_path = os.path.join(tmpdir, "orig/")
+ os.makedirs(orig_full_path)
+
+ data = urandom(1048576)
+ with open(os.path.join(orig_full_path, "data"), "wb") as f:
+ f.write(data)
+
+ binpkg_1 = gpkg(settings, "test", os.path.join(tmpdir, "test-1.gpkg.tar"))
+ binpkg_1.compress(orig_full_path, {})
+
+ with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
+ with tarfile.open(
+ os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
+ ) as tar_2:
+ for f in tar_1.getmembers():
+ if f.name == "Manifest":
+ sig = b"""
+-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA256
+
+
+DATA test/image.tar.zst 1049649 BLAKE2B 3112adba9c09023962f26d9dcbf8e74107c05220f2f29aa2ce894f8a4104c3bb238f87095df73735befcf1e1f6039fc3abf4defa87e68ce80f33dd01e09c055a SHA512 9f584727f2e20a50a30e0077b94082c8c1f517ebfc9978eb3281887e24458108e73d1a2ce82eb0b59f5df7181597e4b0a297ae68bbfb36763aa052e6bdbf2c59
+DATA test/image.tar.zst.sig 833 BLAKE2B 214724ae4ff9198879c8c960fd8167632e27982c2278bb873f195abe75b75afa1ebed4c37ec696f5f5bc35c3a1184b60e0b50d56695b072b254f730db01eddb5 SHA512 67316187da8bb6b7a5f9dc6a42ed5c7d72c6184483a97f23c0bebd8b187ac9268e0409eb233c935101606768718c99eaa5699037d6a68c2d88c9ed5331a3f73c
+-----BEGIN PGP SIGNATURE-----
+
+iNUEARYIAH0WIQSMe+CQzU+/D/DeMitA3PGOlxUHlQUCYVrQal8UgAAAAAAuAChp
+c3N1ZXItZnByQG5vdGF0aW9ucy5vcGVucGdwLmZpZnRoaG9yc2VtYW4ubmV0OEM3
+QkUwOTBDRDRGQkYwRkYwREUzMjJCNDBEQ0YxOEU5NzE1MDc5NQAKCRBA3PGOlxUH
+lbmTAP4jdhMTW6g550/t0V7XcixqVtBockOTln8hZrZIQrjAJAD/caDkxgz5Xl8C
+EP1pgSXXGtlUnv6akg/wueFJKEr9KQs=
+=edEg
+-----END PGP SIGNATURE-----
+"""
+ data = io.BytesIO(sig)
+ f.size = len(sig)
+ tar_2.addfile(f, data)
+ data.close()
+ else:
+ tar_2.addfile(f, tar_1.extractfile(f))
+
+ binpkg_2 = gpkg(settings, "test", os.path.join(tmpdir, "test-2.gpkg.tar"))
+ self.assertRaises(
+ InvalidSignature, binpkg_2.decompress, os.path.join(tmpdir, "test")
+ )
+
+ finally:
+ shutil.rmtree(tmpdir)
+ playground.cleanup()
diff --git a/lib/portage/tests/gpkg/test_gpkg_metadata_update.py b/lib/portage/tests/gpkg/test_gpkg_metadata_update.py
new file mode 100644
index 000000000..2d5d35a98
--- /dev/null
+++ b/lib/portage/tests/gpkg/test_gpkg_metadata_update.py
@@ -0,0 +1,59 @@
+# Copright Gentoo Foundation 2006-2020
+# Portage Unit Testing Functionality
+
+import tempfile
+import sys
+from os import urandom
+
+from portage import os
+from portage import shutil
+from portage.util._compare_files import compare_files
+from portage.tests import TestCase
+from portage.tests.resolver.ResolverPlayground import ResolverPlayground
+from portage.gpkg import gpkg
+
+
+class test_gpkg_metadata_case(TestCase):
+ def test_gpkg_update_metadata(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": ('BINPKG_COMPRESS="gzip"',),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+
+ try:
+ settings = playground.settings
+ orig_full_path = os.path.join(tmpdir, "orig/")
+ os.makedirs(orig_full_path)
+ with open(os.path.join(orig_full_path, "test"), "wb") as test_file:
+ test_file.write(urandom(1048576))
+
+ gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
+ test_gpkg = gpkg(settings, "test", gpkg_file_loc)
+
+ meta = {"test1": b"1234567890", "test2": b"abcdef"}
+
+ test_gpkg.compress(os.path.join(tmpdir, "orig"), meta)
+
+ meta_result = test_gpkg.get_metadata()
+ self.assertEqual(meta, meta_result)
+
+ meta_new = {"test3": b"0987654321", "test4": b"XXXXXXXX"}
+ test_gpkg.update_metadata(meta_new)
+
+ meta_result = test_gpkg.get_metadata()
+ self.assertEqual(meta_new, meta_result)
+
+ test_gpkg.decompress(os.path.join(tmpdir, "test"))
+ r = compare_files(
+ os.path.join(tmpdir, "orig/" + "test"),
+ os.path.join(tmpdir, "test/" + "test"),
+ skipped_types=("atime", "mtime", "ctime"),
+ )
+ self.assertEqual(r, ())
+ finally:
+ shutil.rmtree(tmpdir)
+ playground.cleanup()
diff --git a/lib/portage/tests/gpkg/test_gpkg_metadata_url.py b/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
new file mode 100644
index 000000000..08591715b
--- /dev/null
+++ b/lib/portage/tests/gpkg/test_gpkg_metadata_url.py
@@ -0,0 +1,173 @@
+# Copright Gentoo Foundation 2006-2020
+# Portage Unit Testing Functionality
+
+import io
+import random
+import sys
+import tarfile
+import tempfile
+from functools import partial
+from os import urandom
+
+from portage.gpkg import gpkg
+from portage import os
+from portage import shutil
+from portage.tests import TestCase
+from portage.tests.resolver.ResolverPlayground import ResolverPlayground
+from portage.exception import InvalidSignature
+from portage.gpg import GPG
+
+
+class test_gpkg_metadata_url_case(TestCase):
+ def httpd(self, directory, port):
+ try:
+ import http.server
+ import socketserver
+ except ImportError:
+ self.skipTest("http server not exits")
+
+ Handler = partial(http.server.SimpleHTTPRequestHandler, directory=directory)
+
+ with socketserver.TCPServer(("127.0.0.1", port), Handler) as httpd:
+ httpd.serve_forever()
+
+ def start_http_server(self, directory, port):
+ try:
+ import threading
+ except ImportError:
+ self.skipTest("threading module not exists")
+
+ server = threading.Thread(
+ target=self.httpd, args=(directory, port), daemon=True
+ )
+ server.start()
+ return server
+
+ def test_gpkg_get_metadata_url(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ if sys.version_info.major == 3 and sys.version_info.minor <= 6:
+ self.skipTest("http server not support change root dir")
+
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": (
+ 'BINPKG_COMPRESS="gzip"',
+ 'FEATURES="${FEATURES} -binpkg-signing '
+ '-binpkg-request-signature"',
+ ),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+ try:
+ settings = playground.settings
+ for _ in range(0, 5):
+ port = random.randint(30000, 60000)
+ try:
+ server = self.start_http_server(tmpdir, port)
+ except OSError:
+ continue
+ break
+
+ orig_full_path = os.path.join(tmpdir, "orig/")
+ os.makedirs(orig_full_path)
+
+ with open(os.path.join(orig_full_path, "test"), "wb") as test_file:
+ test_file.write(urandom(1048576))
+
+ gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
+ test_gpkg = gpkg(settings, "test", gpkg_file_loc)
+
+ meta = {
+ "test1": b"{abcdefghijklmnopqrstuvwxyz, 1234567890}",
+ "test2": urandom(102400),
+ }
+
+ test_gpkg.compress(os.path.join(tmpdir, "orig"), meta)
+
+ meta_from_url = test_gpkg.get_metadata_url(
+ "http://127.0.0.1:" + str(port) + "/test.gpkg.tar"
+ )
+
+ self.assertEqual(meta, meta_from_url)
+ finally:
+ shutil.rmtree(tmpdir)
+ playground.cleanup()
+
+ def test_gpkg_get_metadata_url_unknown_signature(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ if sys.version_info.major == 3 and sys.version_info.minor <= 6:
+ self.skipTest("http server not support change root dir")
+
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": (
+ 'BINPKG_COMPRESS="gzip"',
+ 'FEATURES="${FEATURES} binpkg-signing ' 'binpkg-request-signature"',
+ ),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+ try:
+ settings = playground.settings
+ gpg = GPG(settings)
+ gpg.unlock()
+
+ for _ in range(0, 5):
+ port = random.randint(30000, 60000)
+ try:
+ server = self.start_http_server(tmpdir, port)
+ except OSError:
+ continue
+ break
+
+ orig_full_path = os.path.join(tmpdir, "orig/")
+ os.makedirs(orig_full_path)
+
+ with open(os.path.join(orig_full_path, "test"), "wb") as test_file:
+ test_file.write(urandom(1048576))
+
+ gpkg_file_loc = os.path.join(tmpdir, "test-1.gpkg.tar")
+ test_gpkg = gpkg(settings, "test", gpkg_file_loc)
+
+ meta = {
+ "test1": b"{abcdefghijklmnopqrstuvwxyz, 1234567890}",
+ "test2": urandom(102400),
+ }
+
+ test_gpkg.compress(os.path.join(tmpdir, "orig"), meta)
+
+ with tarfile.open(os.path.join(tmpdir, "test-1.gpkg.tar"), "r") as tar_1:
+ with tarfile.open(
+ os.path.join(tmpdir, "test-2.gpkg.tar"), "w"
+ ) as tar_2:
+ for f in tar_1.getmembers():
+ if f.name == "test/metadata.tar.gz":
+ sig = b"""
+-----BEGIN PGP SIGNATURE-----
+
+iHUEABYIAB0WIQRVhCbPGi/rhGTq4nV+k2dcK9uyIgUCXw4ehAAKCRB+k2dcK9uy
+IkCfAP49AOYjzuQPP0n5P0SGCINnAVEXN7QLQ4PurY/lt7cT2gEAq01stXjFhrz5
+87Koh+ND2r5XfQsz3XeBqbb/BpmbEgo=
+=sc5K
+-----END PGP SIGNATURE-----
+"""
+ data = io.BytesIO(sig)
+ f.size = len(sig)
+ tar_2.addfile(f, data)
+ data.close()
+ else:
+ tar_2.addfile(f, tar_1.extractfile(f))
+
+ test_gpkg = gpkg(settings, "test")
+ self.assertRaises(
+ InvalidSignature,
+ test_gpkg.get_metadata_url,
+ "http://127.0.0.1:" + str(port) + "/test-2.gpkg.tar",
+ )
+ finally:
+ shutil.rmtree(tmpdir)
+ playground.cleanup()
diff --git a/lib/portage/tests/gpkg/test_gpkg_path.py b/lib/portage/tests/gpkg/test_gpkg_path.py
new file mode 100644
index 000000000..828233dbd
--- /dev/null
+++ b/lib/portage/tests/gpkg/test_gpkg_path.py
@@ -0,0 +1,390 @@
+# -*- coding: utf-8 -*-
+# Copright Gentoo Foundation 2006
+# Portage Unit Testing Functionality
+
+import tempfile
+import tarfile
+import io
+import sys
+from os import urandom
+
+from portage import os
+from portage import shutil
+from portage.util._compare_files import compare_files
+from portage.tests import TestCase
+from portage.tests.resolver.ResolverPlayground import ResolverPlayground
+from portage.gpkg import gpkg
+
+
+class test_gpkg_path_case(TestCase):
+ def test_gpkg_short_path(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": ('BINPKG_COMPRESS="none"',),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+
+ try:
+ settings = playground.settings
+ path_name = (
+ "aaaabbbb/ccccdddd/eeeeffff/gggghhhh/iiiijjjj/kkkkllll/"
+ "mmmmnnnn/oooopppp/qqqqrrrr/sssstttt/"
+ )
+ orig_full_path = os.path.join(tmpdir, "orig/" + path_name)
+ os.makedirs(orig_full_path)
+ with open(os.path.join(orig_full_path, "test"), "wb") as test_file:
+ test_file.write(urandom(1048576))
+
+ gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
+ test_gpkg = gpkg(settings, "test", gpkg_file_loc)
+
+ check_result = test_gpkg._check_pre_image_files(
+ os.path.join(tmpdir, "orig")
+ )
+ self.assertEqual(check_result, (95, 4, 0, 1048576, 1048576))
+
+ test_gpkg.compress(os.path.join(tmpdir, "orig"), {"meta": "test"})
+ with open(gpkg_file_loc, "rb") as container:
+ # container
+ self.assertEqual(
+ test_gpkg._get_tar_format(container), tarfile.USTAR_FORMAT
+ )
+
+ with tarfile.open(gpkg_file_loc, "r") as container:
+ metadata = io.BytesIO(container.extractfile("test/metadata.tar").read())
+ self.assertEqual(
+ test_gpkg._get_tar_format(metadata), tarfile.USTAR_FORMAT
+ )
+ metadata.close()
+
+ image = io.BytesIO(container.extractfile("test/image.tar").read())
+ self.assertEqual(test_gpkg._get_tar_format(image), tarfile.USTAR_FORMAT)
+ image.close()
+
+ test_gpkg.decompress(os.path.join(tmpdir, "test"))
+ r = compare_files(
+ os.path.join(tmpdir, "orig/" + path_name + "test"),
+ os.path.join(tmpdir, "test/" + path_name + "test"),
+ skipped_types=("atime", "mtime", "ctime"),
+ )
+ self.assertEqual(r, ())
+ finally:
+ shutil.rmtree(tmpdir)
+ playground.cleanup()
+
+ def test_gpkg_long_path(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": ('BINPKG_COMPRESS="none"',),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+
+ try:
+ settings = playground.settings
+
+ path_name = (
+ "aaaabbbb/ccccdddd/eeeeffff/gggghhhh/iiiijjjj/kkkkllll/"
+ "mmmmnnnn/oooopppp/qqqqrrrr/sssstttt/uuuuvvvv/wwwwxxxx/"
+ "yyyyzzzz/00001111/22223333/44445555/66667777/88889999/"
+ "aaaabbbb/ccccdddd/eeeeffff/gggghhhh/iiiijjjj/kkkkllll/"
+ "mmmmnnnn/oooopppp/qqqqrrrr/sssstttt/uuuuvvvv/wwwwxxxx/"
+ "yyyyzzzz/00001111/22223333/44445555/66667777/88889999/"
+ )
+ orig_full_path = os.path.join(tmpdir, "orig/" + path_name)
+ os.makedirs(orig_full_path)
+ with open(os.path.join(orig_full_path, "test"), "wb") as test_file:
+ test_file.write(urandom(1048576))
+
+ gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
+ test_gpkg = gpkg(settings, "test", gpkg_file_loc)
+
+ check_result = test_gpkg._check_pre_image_files(
+ os.path.join(tmpdir, "orig")
+ )
+ self.assertEqual(check_result, (329, 4, 0, 1048576, 1048576))
+
+ test_gpkg.compress(os.path.join(tmpdir, "orig"), {"meta": "test"})
+ with open(gpkg_file_loc, "rb") as container:
+ # container
+ self.assertEqual(
+ test_gpkg._get_tar_format(container), tarfile.USTAR_FORMAT
+ )
+
+ with tarfile.open(gpkg_file_loc, "r") as container:
+ metadata = io.BytesIO(container.extractfile("test/metadata.tar").read())
+ self.assertEqual(
+ test_gpkg._get_tar_format(metadata), tarfile.USTAR_FORMAT
+ )
+ metadata.close()
+
+ image = io.BytesIO(container.extractfile("test/image.tar").read())
+ self.assertEqual(test_gpkg._get_tar_format(image), tarfile.GNU_FORMAT)
+ image.close()
+
+ test_gpkg.decompress(os.path.join(tmpdir, "test"))
+ r = compare_files(
+ os.path.join(tmpdir, "orig/" + path_name + "test"),
+ os.path.join(tmpdir, "test/" + path_name + "test"),
+ skipped_types=("atime", "mtime", "ctime"),
+ )
+ self.assertEqual(r, ())
+ finally:
+ shutil.rmtree(tmpdir)
+ playground.cleanup()
+
+ def test_gpkg_non_ascii_path(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": ('BINPKG_COMPRESS="none"',),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+
+ try:
+ settings = playground.settings
+
+ path_name = "中文测试/日本語テスト/한국어시험/"
+ orig_full_path = os.path.join(tmpdir, "orig/" + path_name)
+ os.makedirs(orig_full_path)
+ with open(os.path.join(orig_full_path, "test"), "wb") as test_file:
+ test_file.write(urandom(1048576))
+
+ gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
+ test_gpkg = gpkg(settings, "test", gpkg_file_loc)
+
+ check_result = test_gpkg._check_pre_image_files(
+ os.path.join(tmpdir, "orig")
+ )
+ self.assertEqual(check_result, (53, 4, 0, 1048576, 1048576))
+
+ test_gpkg.compress(os.path.join(tmpdir, "orig"), {"meta": "test"})
+ with open(gpkg_file_loc, "rb") as container:
+ # container
+ self.assertEqual(
+ test_gpkg._get_tar_format(container), tarfile.USTAR_FORMAT
+ )
+
+ with tarfile.open(gpkg_file_loc, "r") as container:
+ metadata = io.BytesIO(container.extractfile("test/metadata.tar").read())
+ self.assertEqual(
+ test_gpkg._get_tar_format(metadata), tarfile.USTAR_FORMAT
+ )
+ metadata.close()
+
+ image = io.BytesIO(container.extractfile("test/image.tar").read())
+ self.assertEqual(test_gpkg._get_tar_format(image), tarfile.USTAR_FORMAT)
+ image.close()
+
+ test_gpkg.decompress(os.path.join(tmpdir, "test"))
+ r = compare_files(
+ os.path.join(tmpdir, "orig/" + path_name + "test"),
+ os.path.join(tmpdir, "test/" + path_name + "test"),
+ skipped_types=("atime", "mtime", "ctime"),
+ )
+ self.assertEqual(r, ())
+ finally:
+ shutil.rmtree(tmpdir)
+ playground.cleanup()
+
+ def test_gpkg_symlink_path(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": ('BINPKG_COMPRESS="none"',),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+
+ try:
+ settings = playground.settings
+
+ orig_full_path = os.path.join(tmpdir, "orig/")
+ os.makedirs(orig_full_path)
+ os.symlink(
+ "aaaabbbb/ccccdddd/eeeeffff/gggghhhh/iiiijjjj/kkkkllll/"
+ "mmmmnnnn/oooopppp/qqqqrrrr/sssstttt/uuuuvvvv/wwwwxxxx/"
+ "yyyyzzzz/00001111/22223333/44445555/66667777/88889999/test",
+ os.path.join(orig_full_path, "a_long_symlink"),
+ )
+
+ gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
+ test_gpkg = gpkg(settings, "test", gpkg_file_loc)
+
+ check_result = test_gpkg._check_pre_image_files(
+ os.path.join(tmpdir, "orig")
+ )
+ self.assertEqual(check_result, (0, 14, 166, 0, 0))
+
+ test_gpkg.compress(os.path.join(tmpdir, "orig"), {"meta": "test"})
+ with open(gpkg_file_loc, "rb") as container:
+ # container
+ self.assertEqual(
+ test_gpkg._get_tar_format(container), tarfile.USTAR_FORMAT
+ )
+
+ with tarfile.open(gpkg_file_loc, "r") as container:
+ metadata = io.BytesIO(container.extractfile("test/metadata.tar").read())
+ self.assertEqual(
+ test_gpkg._get_tar_format(metadata), tarfile.USTAR_FORMAT
+ )
+ metadata.close()
+
+ image = io.BytesIO(container.extractfile("test/image.tar").read())
+ self.assertEqual(test_gpkg._get_tar_format(image), tarfile.GNU_FORMAT)
+ image.close()
+
+ test_gpkg.decompress(os.path.join(tmpdir, "test"))
+ r = compare_files(
+ os.path.join(tmpdir, "orig/", "a_long_symlink"),
+ os.path.join(tmpdir, "test/", "a_long_symlink"),
+ skipped_types=("atime", "mtime", "ctime"),
+ )
+ self.assertEqual(r, ())
+ finally:
+ shutil.rmtree(tmpdir)
+ playground.cleanup()
+
+ def test_gpkg_long_hardlink_path(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": ('BINPKG_COMPRESS="none"',),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+
+ try:
+ settings = playground.settings
+
+ path_name = (
+ "aaaabbbb/ccccdddd/eeeeffff/gggghhhh/iiiijjjj/kkkkllll/"
+ "mmmmnnnn/oooopppp/qqqqrrrr/sssstttt/uuuuvvvv/wwwwxxxx/"
+ )
+ file_name = (
+ "test-A-B-C-D-E-F-G-H-I-J-K-L-M-N-O-P-Q-R-S-T-U-V-W-X-Y-Z"
+ "A-B-C-D-E-F-G-H-I-J-K-L-M-N-O-P-Q-R-S-T-U-V-W-X-Y-Z"
+ "A-B-C-D-E-F-G-H-I-J-K-L-M-N-O-P-Q-R-S-T-U-V-W-X-Y-Z"
+ )
+ orig_full_path = os.path.join(tmpdir, "orig", path_name)
+ os.makedirs(orig_full_path)
+ with open(os.path.join(orig_full_path, "test"), "wb") as test_file:
+ test_file.write(urandom(1048576))
+
+ os.link(
+ os.path.join(orig_full_path, "test"),
+ os.path.join(orig_full_path, file_name),
+ )
+
+ gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
+ test_gpkg = gpkg(settings, "test", gpkg_file_loc)
+
+ check_result = test_gpkg._check_pre_image_files(
+ os.path.join(tmpdir, "orig")
+ )
+ self.assertEqual(check_result, (113, 158, 272, 1048576, 2097152))
+
+ test_gpkg.compress(os.path.join(tmpdir, "orig"), {"meta": "test"})
+ with open(gpkg_file_loc, "rb") as container:
+ # container
+ self.assertEqual(
+ test_gpkg._get_tar_format(container), tarfile.USTAR_FORMAT
+ )
+
+ with tarfile.open(gpkg_file_loc, "r") as container:
+ metadata = io.BytesIO(container.extractfile("test/metadata.tar").read())
+ self.assertEqual(
+ test_gpkg._get_tar_format(metadata), tarfile.USTAR_FORMAT
+ )
+ metadata.close()
+
+ image = io.BytesIO(container.extractfile("test/image.tar").read())
+ self.assertEqual(test_gpkg._get_tar_format(image), tarfile.GNU_FORMAT)
+ image.close()
+
+ test_gpkg.decompress(os.path.join(tmpdir, "test"))
+ r = compare_files(
+ os.path.join(tmpdir, "orig", path_name, file_name),
+ os.path.join(tmpdir, "test", path_name, file_name),
+ skipped_types=("atime", "mtime", "ctime"),
+ )
+ self.assertEqual(r, ())
+ finally:
+ shutil.rmtree(tmpdir)
+
+ def test_gpkg_long_filename(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": ('BINPKG_COMPRESS="none"',),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+
+ try:
+ settings = playground.settings
+ path_name = "aaaabbbb/ccccdddd/eeeeffff/gggghhhh/iiiijjjj/kkkkllll/"
+ file_name = (
+ "test1234567890"
+ "A-B-C-D-E-F-G-H-I-J-K-L-M-N-O-P-Q-R-S-T-U-V-W-X-Y-Z"
+ "A-B-C-D-E-F-G-H-I-J-K-L-M-N-O-P-Q-R-S-T-U-V-W-X-Y-Z"
+ "A-B-C-D-E-F-G-H-I-J-K-L-M-N-O-P-Q-R-S-T-U-V-W-X-Y-Z"
+ )
+
+ orig_full_path = os.path.join(tmpdir, "orig/" + path_name)
+ os.makedirs(orig_full_path)
+ with open(os.path.join(orig_full_path, file_name), "wb") as test_file:
+ test_file.write(urandom(1048576))
+
+ gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
+ test_gpkg = gpkg(settings, "test", gpkg_file_loc)
+
+ check_result = test_gpkg._check_pre_image_files(
+ os.path.join(tmpdir, "orig")
+ )
+ self.assertEqual(check_result, (59, 167, 0, 1048576, 1048576))
+
+ test_gpkg.compress(os.path.join(tmpdir, "orig"), {"meta": "test"})
+ with open(gpkg_file_loc, "rb") as container:
+ # container
+ self.assertEqual(
+ test_gpkg._get_tar_format(container), tarfile.USTAR_FORMAT
+ )
+
+ with tarfile.open(gpkg_file_loc, "r") as container:
+ metadata = io.BytesIO(container.extractfile("test/metadata.tar").read())
+ self.assertEqual(
+ test_gpkg._get_tar_format(metadata), tarfile.USTAR_FORMAT
+ )
+ metadata.close()
+
+ image = io.BytesIO(container.extractfile("test/image.tar").read())
+ self.assertEqual(test_gpkg._get_tar_format(image), tarfile.GNU_FORMAT)
+ image.close()
+
+ test_gpkg.decompress(os.path.join(tmpdir, "test"))
+ r = compare_files(
+ os.path.join(tmpdir, "orig", path_name, file_name),
+ os.path.join(tmpdir, "test", path_name, file_name),
+ skipped_types=("atime", "mtime", "ctime"),
+ )
+ self.assertEqual(r, ())
+ finally:
+ shutil.rmtree(tmpdir)
+ playground.cleanup()
diff --git a/lib/portage/tests/gpkg/test_gpkg_size.py b/lib/portage/tests/gpkg/test_gpkg_size.py
new file mode 100644
index 000000000..a38621e70
--- /dev/null
+++ b/lib/portage/tests/gpkg/test_gpkg_size.py
@@ -0,0 +1,58 @@
+# Copright Gentoo Foundation 2006-2020
+# Portage Unit Testing Functionality
+
+import tempfile
+import tarfile
+import sys
+
+from portage import os, shutil
+from portage.tests import TestCase
+from portage.tests.resolver.ResolverPlayground import ResolverPlayground
+from portage.gpkg import gpkg
+
+
+class test_gpkg_large_size_case(TestCase):
+ def test_gpkg_large_size(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ playground = ResolverPlayground(
+ user_config={
+ "make.conf": ('BINPKG_COMPRESS="gzip"',),
+ }
+ )
+ tmpdir = tempfile.mkdtemp()
+
+ try:
+ settings = playground.settings
+
+ orig_full_path = os.path.join(tmpdir, "orig/")
+ os.makedirs(orig_full_path)
+ # Check if filesystem support sparse file
+ with open(os.path.join(orig_full_path, "test"), "wb") as test_file:
+ test_file.truncate(1048576)
+
+ if os.stat(os.path.join(orig_full_path, "test")).st_blocks != 0:
+ self.skipTest("Filesystem does not support sparse file")
+
+ with open(os.path.join(orig_full_path, "test"), "wb") as test_file:
+ test_file.truncate(10737418240)
+
+ gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
+ test_gpkg = gpkg(settings, "test", gpkg_file_loc)
+
+ check_result = test_gpkg._check_pre_image_files(
+ os.path.join(tmpdir, "orig")
+ )
+ self.assertEqual(check_result, (0, 4, 0, 10737418240, 10737418240))
+
+ test_gpkg.compress(os.path.join(tmpdir, "orig"), {"meta": "test"})
+
+ with open(gpkg_file_loc, "rb") as container:
+ # container
+ self.assertEqual(
+ test_gpkg._get_tar_format(container), tarfile.GNU_FORMAT
+ )
+ finally:
+ shutil.rmtree(tmpdir)
+ playground.cleanup()
diff --git a/lib/portage/tests/gpkg/test_gpkg_stream.py b/lib/portage/tests/gpkg/test_gpkg_stream.py
new file mode 100644
index 000000000..66471e01b
--- /dev/null
+++ b/lib/portage/tests/gpkg/test_gpkg_stream.py
@@ -0,0 +1,112 @@
+# Copright Gentoo Foundation 2006-2020
+# Portage Unit Testing Functionality
+
+import sys
+import tempfile
+import io
+import tarfile
+from os import urandom
+
+import portage.gpkg
+from portage import os
+from portage import shutil
+from portage.tests import TestCase
+from portage.exception import CompressorOperationFailed
+
+
+class test_gpkg_stream_case(TestCase):
+ def test_gpkg_stream_reader(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ data = urandom(1048576)
+ data_io = io.BytesIO(data)
+ data_io.seek(0)
+ with portage.gpkg.tar_stream_reader(data_io, ["cat"]) as test_reader:
+ data2 = test_reader.read()
+ data_io.close()
+ self.assertEqual(data, data2)
+
+ def test_gpkg_stream_reader_without_cmd(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ data = urandom(1048576)
+ data_io = io.BytesIO(data)
+ data_io.seek(0)
+ with portage.gpkg.tar_stream_reader(data_io) as test_reader:
+ data2 = test_reader.read()
+ data_io.close()
+ self.assertEqual(data, data2)
+
+ def test_gpkg_stream_reader_kill(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ data = urandom(1048576)
+ data_io = io.BytesIO(data)
+ data_io.seek(0)
+ with portage.gpkg.tar_stream_reader(data_io, ["cat"]) as test_reader:
+ try:
+ test_reader.kill()
+ except CompressorOperationFailed:
+ pass
+ data_io.close()
+ self.assertNotEqual(test_reader.proc.poll(), None)
+
+ def test_gpkg_stream_reader_kill_without_cmd(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ data = urandom(1048576)
+ data_io = io.BytesIO(data)
+ data_io.seek(0)
+ with portage.gpkg.tar_stream_reader(data_io) as test_reader:
+ test_reader.kill()
+ data_io.close()
+ self.assertEqual(test_reader.proc, None)
+
+ def test_gpkg_stream_writer(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ tmpdir = tempfile.mkdtemp()
+ try:
+ gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
+ data = urandom(1048576)
+ with tarfile.open(gpkg_file_loc, "w") as test_tar:
+ test_tarinfo = tarfile.TarInfo("test")
+ with portage.gpkg.tar_stream_writer(
+ test_tarinfo, test_tar, tarfile.USTAR_FORMAT, ["cat"]
+ ) as test_writer:
+ test_writer.write(data)
+
+ with tarfile.open(gpkg_file_loc, "r") as test_tar:
+ test_tarinfo = test_tar.getmember("test")
+ data2 = test_tar.extractfile(test_tarinfo).read()
+ self.assertEqual(data, data2)
+ finally:
+ shutil.rmtree(tmpdir)
+
+ def test_gpkg_stream_writer_without_cmd(self):
+ if sys.version_info.major < 3:
+ self.skipTest("Not support Python 2")
+
+ tmpdir = tempfile.mkdtemp()
+
+ try:
+ gpkg_file_loc = os.path.join(tmpdir, "test.gpkg.tar")
+ data = urandom(1048576)
+ with tarfile.open(gpkg_file_loc, "w") as test_tar:
+ test_tarinfo = tarfile.TarInfo("test")
+ with portage.gpkg.tar_stream_writer(
+ test_tarinfo, test_tar, tarfile.USTAR_FORMAT
+ ) as test_writer:
+ test_writer.write(data)
+
+ with tarfile.open(gpkg_file_loc, "r") as test_tar:
+ test_tarinfo = test_tar.getmember("test")
+ data2 = test_tar.extractfile(test_tarinfo).read()
+ self.assertEqual(data, data2)
+ finally:
+ shutil.rmtree(tmpdir)
diff --git a/lib/portage/tests/resolver/ResolverPlayground.py b/lib/portage/tests/resolver/ResolverPlayground.py
index fdd0714e6..fa8b0cc76 100644
--- a/lib/portage/tests/resolver/ResolverPlayground.py
+++ b/lib/portage/tests/resolver/ResolverPlayground.py
@@ -13,6 +13,7 @@ from portage.const import (
GLOBAL_CONFIG_PATH,
PORTAGE_BIN_PATH,
USER_CONFIG_PATH,
+ SUPPORTED_GENTOO_BINPKG_FORMATS,
)
from portage.process import find_binary
from portage.dep import Atom, _repo_separator
@@ -23,6 +24,8 @@ from portage._sets.base import InternalPackageSet
from portage.tests import cnf_path
from portage.util import ensure_dirs, normalize_path
from portage.versions import catsplit
+from portage.exception import InvalidBinaryPackageFormat
+from portage.gpg import GPG
import _emerge
from _emerge.actions import _calc_depclean
@@ -161,6 +164,7 @@ class ResolverPlayground:
"egrep",
"env",
"find",
+ "flock",
"grep",
"head",
"install",
@@ -225,7 +229,6 @@ class ResolverPlayground:
self._create_distfiles(distfiles)
self._create_ebuilds(ebuilds)
- self._create_binpkgs(binpkgs)
self._create_installed(installed)
self._create_profile(
ebuilds, eclasses, installed, profile, repo_configs, user_config, sets
@@ -234,6 +237,8 @@ class ResolverPlayground:
self.settings, self.trees = self._load_config()
+ self.gpg = None
+ self._create_binpkgs(binpkgs)
self._create_ebuild_manifests(ebuilds)
portage.util.noiselimit = 0
@@ -341,6 +346,13 @@ class ResolverPlayground:
# a dict.
items = getattr(binpkgs, "items", None)
items = items() if items is not None else binpkgs
+ binpkg_format = self.settings.get(
+ "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]
+ )
+ if binpkg_format == "gpkg":
+ if self.gpg is None:
+ self.gpg = GPG(self.settings)
+ self.gpg.unlock()
for cpv, metadata in items:
a = Atom("=" + cpv, allow_repo=True)
repo = a.repo
@@ -356,19 +368,38 @@ class ResolverPlayground:
metadata["repository"] = repo
metadata["CATEGORY"] = cat
metadata["PF"] = pf
+ metadata["BINPKG_FORMAT"] = binpkg_format
repo_dir = self.pkgdir
category_dir = os.path.join(repo_dir, cat)
if "BUILD_ID" in metadata:
- binpkg_path = os.path.join(
- category_dir, pn, "%s-%s.xpak" % (pf, metadata["BUILD_ID"])
- )
+ if binpkg_format == "xpak":
+ binpkg_path = os.path.join(
+ category_dir, pn, "%s-%s.xpak" % (pf, metadata["BUILD_ID"])
+ )
+ elif binpkg_format == "gpkg":
+ binpkg_path = os.path.join(
+ category_dir, pn, "%s-%s.gpkg.tar" % (pf, metadata["BUILD_ID"])
+ )
+ else:
+ raise InvalidBinaryPackageFormat(binpkg_format)
else:
- binpkg_path = os.path.join(category_dir, pf + ".tbz2")
+ if binpkg_format == "xpak":
+ binpkg_path = os.path.join(category_dir, pf + ".tbz2")
+ elif binpkg_format == "gpkg":
+ binpkg_path = os.path.join(category_dir, pf + ".gpkg.tar")
+ else:
+ raise InvalidBinaryPackageFormat(binpkg_format)
ensure_dirs(os.path.dirname(binpkg_path))
- t = portage.xpak.tbz2(binpkg_path)
- t.recompose_mem(portage.xpak.xpak_mem(metadata))
+ if binpkg_format == "xpak":
+ t = portage.xpak.tbz2(binpkg_path)
+ t.recompose_mem(portage.xpak.xpak_mem(metadata))
+ elif binpkg_format == "gpkg":
+ t = portage.gpkg.gpkg(self.settings, a.cpv, binpkg_path)
+ t.compress(os.path.dirname(binpkg_path), metadata)
+ else:
+ raise InvalidBinaryPackageFormat(binpkg_format)
def _create_installed(self, installed):
for cpv in installed:
@@ -545,11 +576,19 @@ class ResolverPlayground:
sub_profile_dir, os.path.join(user_config_dir, "make.profile")
)
+ gpg_test_path = os.environ["PORTAGE_GNUPGHOME"]
+
make_conf = {
"ACCEPT_KEYWORDS": "x86",
+ "BINPKG_GPG_SIGNING_BASE_COMMAND": f"flock {gpg_test_path}/portage-binpkg-gpg.lock /usr/bin/gpg --sign --armor --yes --pinentry-mode loopback --passphrase GentooTest [PORTAGE_CONFIG]",
+ "BINPKG_GPG_SIGNING_GPG_HOME": gpg_test_path,
+ "BINPKG_GPG_SIGNING_KEY": "0x5D90EA06352177F6",
+ "BINPKG_GPG_VERIFY_GPG_HOME": gpg_test_path,
"CLEAN_DELAY": "0",
"DISTDIR": self.distdir,
"EMERGE_WARNING_DELAY": "0",
+ "FEATURES": "${FEATURES} binpkg-signing binpkg-request-signature "
+ "gpg-keepalive",
"PKGDIR": self.pkgdir,
"PORTAGE_INST_GID": str(portage.data.portage_gid),
"PORTAGE_INST_UID": str(portage.data.portage_uid),
@@ -742,6 +781,8 @@ class ResolverPlayground:
return
def cleanup(self):
+ if self.gpg is not None:
+ self.gpg.stop()
for eroot in self.trees:
portdb = self.trees[eroot]["porttree"].dbapi
portdb.close_caches()
diff --git a/lib/portage/tests/resolver/binpkg_multi_instance/test_build_id_profile_format.py b/lib/portage/tests/resolver/binpkg_multi_instance/test_build_id_profile_format.py
index b311961d6..4dc83e843 100644
--- a/lib/portage/tests/resolver/binpkg_multi_instance/test_build_id_profile_format.py
+++ b/lib/portage/tests/resolver/binpkg_multi_instance/test_build_id_profile_format.py
@@ -1,11 +1,16 @@
# Copyright 2015-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class BuildIdProfileFormatTestCase(TestCase):
@@ -139,21 +144,30 @@ class BuildIdProfileFormatTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- debug=False,
- binpkgs=binpkgs,
- ebuilds=ebuilds,
- installed=installed,
- repo_configs=repo_configs,
- profile=profile,
- user_config=user_config,
- world=world,
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- # Disable debug so that cleanup works.
- # playground.debug = False
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ _user_config = user_config.copy()
+ _user_config["make.conf"] += ('BINPKG_FORMAT="%s"' % binpkg_format,)
+ playground = ResolverPlayground(
+ debug=False,
+ binpkgs=binpkgs,
+ ebuilds=ebuilds,
+ installed=installed,
+ repo_configs=repo_configs,
+ profile=profile,
+ user_config=_user_config,
+ world=world,
+ )
+
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ # Disable debug so that cleanup works.
+ # playground.debug = False
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/binpkg_multi_instance/test_rebuilt_binaries.py b/lib/portage/tests/resolver/binpkg_multi_instance/test_rebuilt_binaries.py
index c854604c1..854dee458 100644
--- a/lib/portage/tests/resolver/binpkg_multi_instance/test_rebuilt_binaries.py
+++ b/lib/portage/tests/resolver/binpkg_multi_instance/test_rebuilt_binaries.py
@@ -1,11 +1,16 @@
# Copyright 2015 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class RebuiltBinariesCase(TestCase):
@@ -99,18 +104,27 @@ class RebuiltBinariesCase(TestCase):
),
)
- playground = ResolverPlayground(
- debug=False,
- binpkgs=binpkgs,
- installed=installed,
- user_config=user_config,
- world=world,
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- # Disable debug so that cleanup works.
- # playground.debug = False
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ _user_config = user_config.copy()
+ _user_config["make.conf"] += ('BINPKG_FORMAT="%s"' % binpkg_format,)
+ playground = ResolverPlayground(
+ debug=False,
+ binpkgs=binpkgs,
+ installed=installed,
+ user_config=_user_config,
+ world=world,
+ )
+
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ # Disable debug so that cleanup works.
+ # playground.debug = False
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/soname/test_autounmask.py b/lib/portage/tests/resolver/soname/test_autounmask.py
index 3ee0be8d0..e324c9392 100644
--- a/lib/portage/tests/resolver/soname/test_autounmask.py
+++ b/lib/portage/tests/resolver/soname/test_autounmask.py
@@ -1,11 +1,16 @@
# Copyright 2015 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class SonameAutoUnmaskTestCase(TestCase):
@@ -85,13 +90,26 @@ class SonameAutoUnmaskTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- binpkgs=binpkgs, installed=installed, world=world, debug=False
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.debug = False
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ binpkgs=binpkgs,
+ installed=installed,
+ world=world,
+ debug=False,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.debug = False
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/soname/test_downgrade.py b/lib/portage/tests/resolver/soname/test_downgrade.py
index b683745e0..c601b6381 100644
--- a/lib/portage/tests/resolver/soname/test_downgrade.py
+++ b/lib/portage/tests/resolver/soname/test_downgrade.py
@@ -1,11 +1,16 @@
# Copyright 2015 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class SonameDowngradeTestCase(TestCase):
@@ -125,22 +130,29 @@ class SonameDowngradeTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- binpkgs=binpkgs,
- ebuilds=ebuilds,
- installed=installed,
- user_config=user_config,
- world=world,
- debug=False,
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- # Disable debug so that cleanup works.
- playground.debug = False
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ user_config["make.conf"] = ('BINPKG_FORMAT="%s"' % binpkg_format,)
+ playground = ResolverPlayground(
+ binpkgs=binpkgs,
+ ebuilds=ebuilds,
+ installed=installed,
+ user_config=user_config,
+ world=world,
+ debug=False,
+ )
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ # Disable debug so that cleanup works.
+ playground.debug = False
+ playground.cleanup()
def testTwoSlots(self):
@@ -217,19 +229,27 @@ class SonameDowngradeTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- ebuilds=ebuilds,
- binpkgs=binpkgs,
- installed=installed,
- user_config=user_config,
- world=world,
- debug=False,
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- # Disable debug so that cleanup works.
- playground.debug = False
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ user_config["make.conf"] = ('BINPKG_FORMAT="%s"' % binpkg_format,)
+ playground = ResolverPlayground(
+ ebuilds=ebuilds,
+ binpkgs=binpkgs,
+ installed=installed,
+ user_config=user_config,
+ world=world,
+ debug=False,
+ )
+
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ # Disable debug so that cleanup works.
+ playground.debug = False
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/soname/test_or_choices.py b/lib/portage/tests/resolver/soname/test_or_choices.py
index c636726f3..dcdcf57e3 100644
--- a/lib/portage/tests/resolver/soname/test_or_choices.py
+++ b/lib/portage/tests/resolver/soname/test_or_choices.py
@@ -1,11 +1,16 @@
# Copyright 2015 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class SonameOrChoicesTestCase(TestCase):
@@ -83,14 +88,26 @@ class SonameOrChoicesTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- debug=False, binpkgs=binpkgs, installed=installed, world=world
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- # Disable debug so that cleanup works.
- playground.debug = False
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ debug=False,
+ binpkgs=binpkgs,
+ installed=installed,
+ world=world,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ # Disable debug so that cleanup works.
+ playground.debug = False
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/soname/test_reinstall.py b/lib/portage/tests/resolver/soname/test_reinstall.py
index f4616f9dd..68c842af1 100644
--- a/lib/portage/tests/resolver/soname/test_reinstall.py
+++ b/lib/portage/tests/resolver/soname/test_reinstall.py
@@ -1,11 +1,16 @@
# Copyright 2015 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class SonameReinstallTestCase(TestCase):
@@ -72,14 +77,27 @@ class SonameReinstallTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- debug=False, binpkgs=binpkgs, installed=installed, world=world
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- # Disable debug so that cleanup works.
- playground.debug = False
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ debug=False,
+ binpkgs=binpkgs,
+ installed=installed,
+ world=world,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ # Disable debug so that cleanup works.
+ playground.debug = False
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/soname/test_skip_update.py b/lib/portage/tests/resolver/soname/test_skip_update.py
index 336bfac4f..a515a5252 100644
--- a/lib/portage/tests/resolver/soname/test_skip_update.py
+++ b/lib/portage/tests/resolver/soname/test_skip_update.py
@@ -1,11 +1,16 @@
# Copyright 2015 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class SonameSkipUpdateTestCase(TestCase):
@@ -71,14 +76,26 @@ class SonameSkipUpdateTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- debug=False, binpkgs=binpkgs, installed=installed, world=world
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- # Disable debug so that cleanup works.
- playground.debug = False
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ debug=False,
+ binpkgs=binpkgs,
+ installed=installed,
+ world=world,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ # Disable debug so that cleanup works.
+ playground.debug = False
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/soname/test_slot_conflict_reinstall.py b/lib/portage/tests/resolver/soname/test_slot_conflict_reinstall.py
index 39430ae41..027cadc83 100644
--- a/lib/portage/tests/resolver/soname/test_slot_conflict_reinstall.py
+++ b/lib/portage/tests/resolver/soname/test_slot_conflict_reinstall.py
@@ -1,11 +1,16 @@
# Copyright 2015 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class SonameSlotConflictReinstallTestCase(TestCase):
@@ -80,16 +85,28 @@ class SonameSlotConflictReinstallTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- binpkgs=binpkgs, installed=installed, world=world, debug=False
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.debug = False
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ binpkgs=binpkgs,
+ installed=installed,
+ world=world,
+ debug=False,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.debug = False
+ playground.cleanup()
def testSonameSlotConflictMassRebuild(self):
"""
@@ -159,16 +176,29 @@ class SonameSlotConflictReinstallTestCase(TestCase):
world = []
- playground = ResolverPlayground(
- binpkgs=binpkgs, installed=installed, world=world, debug=False
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.debug = False
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ binpkgs=binpkgs,
+ installed=installed,
+ world=world,
+ debug=False,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.debug = False
+ playground.cleanup()
def testSonameSlotConflictForgottenChild(self):
"""
@@ -242,16 +272,29 @@ class SonameSlotConflictReinstallTestCase(TestCase):
world = ["app-misc/A"]
- playground = ResolverPlayground(
- binpkgs=binpkgs, installed=installed, world=world, debug=False
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.debug = False
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ binpkgs=binpkgs,
+ installed=installed,
+ world=world,
+ debug=False,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.debug = False
+ playground.cleanup()
def testSonameSlotConflictMixedDependencies(self):
"""
@@ -316,13 +359,25 @@ class SonameSlotConflictReinstallTestCase(TestCase):
world = []
- playground = ResolverPlayground(
- binpkgs=binpkgs, installed=installed, world=world, debug=False
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.debug = False
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ binpkgs=binpkgs,
+ installed=installed,
+ world=world,
+ debug=False,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.debug = False
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/soname/test_slot_conflict_update.py b/lib/portage/tests/resolver/soname/test_slot_conflict_update.py
index 0541a185e..dd763caef 100644
--- a/lib/portage/tests/resolver/soname/test_slot_conflict_update.py
+++ b/lib/portage/tests/resolver/soname/test_slot_conflict_update.py
@@ -1,11 +1,16 @@
# Copyright 2015 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class SonameSlotConflictUpdateTestCase(TestCase):
@@ -88,13 +93,26 @@ class SonameSlotConflictUpdateTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- binpkgs=binpkgs, installed=installed, world=world, debug=False
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.debug = False
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ binpkgs=binpkgs,
+ installed=installed,
+ world=world,
+ debug=False,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.debug = False
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/soname/test_soname_provided.py b/lib/portage/tests/resolver/soname/test_soname_provided.py
index 3cd9f1423..6a9ee76ba 100644
--- a/lib/portage/tests/resolver/soname/test_soname_provided.py
+++ b/lib/portage/tests/resolver/soname/test_soname_provided.py
@@ -1,11 +1,16 @@
# Copyright 2015 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class SonameProvidedTestCase(TestCase):
@@ -62,18 +67,28 @@ class SonameProvidedTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- binpkgs=binpkgs,
- debug=False,
- profile=profile,
- installed=installed,
- world=world,
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- # Disable debug so that cleanup works.
- playground.debug = False
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ binpkgs=binpkgs,
+ debug=False,
+ profile=profile,
+ installed=installed,
+ world=world,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ # Disable debug so that cleanup works.
+ playground.debug = False
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/soname/test_unsatisfiable.py b/lib/portage/tests/resolver/soname/test_unsatisfiable.py
index a8d2e10db..75d50c10f 100644
--- a/lib/portage/tests/resolver/soname/test_unsatisfiable.py
+++ b/lib/portage/tests/resolver/soname/test_unsatisfiable.py
@@ -1,11 +1,16 @@
# Copyright 2015 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class SonameUnsatisfiableTestCase(TestCase):
@@ -57,14 +62,27 @@ class SonameUnsatisfiableTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- binpkgs=binpkgs, debug=False, installed=installed, world=world
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- # Disable debug so that cleanup works.
- playground.debug = False
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ binpkgs=binpkgs,
+ debug=False,
+ installed=installed,
+ world=world,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ # Disable debug so that cleanup works.
+ playground.debug = False
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/soname/test_unsatisfied.py b/lib/portage/tests/resolver/soname/test_unsatisfied.py
index 955d5d75b..2e0fe6e7f 100644
--- a/lib/portage/tests/resolver/soname/test_unsatisfied.py
+++ b/lib/portage/tests/resolver/soname/test_unsatisfied.py
@@ -1,11 +1,16 @@
# Copyright 2015 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class SonameUnsatisfiedTestCase(TestCase):
@@ -70,14 +75,27 @@ class SonameUnsatisfiedTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- binpkgs=binpkgs, debug=False, installed=installed, world=world
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- # Disable debug so that cleanup works.
- playground.debug = False
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ binpkgs=binpkgs,
+ debug=False,
+ installed=installed,
+ world=world,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ # Disable debug so that cleanup works.
+ playground.debug = False
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/test_autounmask_binpkg_use.py b/lib/portage/tests/resolver/test_autounmask_binpkg_use.py
index e2164f0b1..682732611 100644
--- a/lib/portage/tests/resolver/test_autounmask_binpkg_use.py
+++ b/lib/portage/tests/resolver/test_autounmask_binpkg_use.py
@@ -1,11 +1,16 @@
# Copyright 2017 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class AutounmaskBinpkgUseTestCase(TestCase):
@@ -55,13 +60,26 @@ class AutounmaskBinpkgUseTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- ebuilds=ebuilds, binpkgs=binpkgs, installed=installed, debug=False
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.debug = False
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ ebuilds=ebuilds,
+ binpkgs=binpkgs,
+ installed=installed,
+ debug=False,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.debug = False
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/test_bdeps.py b/lib/portage/tests/resolver/test_bdeps.py
index ce1f8e0d5..a1b987eca 100644
--- a/lib/portage/tests/resolver/test_bdeps.py
+++ b/lib/portage/tests/resolver/test_bdeps.py
@@ -1,11 +1,16 @@
# Copyright 2017 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
+from portage.output import colorize
class BdepsTestCase(TestCase):
@@ -183,18 +188,27 @@ class BdepsTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- debug=False,
- ebuilds=ebuilds,
- installed=installed,
- binpkgs=binpkgs,
- world=world,
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- # Disable debug so that cleanup works.
- playground.debug = False
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ debug=False,
+ ebuilds=ebuilds,
+ installed=installed,
+ binpkgs=binpkgs,
+ world=world,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ # Disable debug so that cleanup works.
+ playground.debug = False
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/test_binary_pkg_ebuild_visibility.py b/lib/portage/tests/resolver/test_binary_pkg_ebuild_visibility.py
index 9a6a5417a..10a292abd 100644
--- a/lib/portage/tests/resolver/test_binary_pkg_ebuild_visibility.py
+++ b/lib/portage/tests/resolver/test_binary_pkg_ebuild_visibility.py
@@ -1,11 +1,16 @@
# Copyright 2017 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class BinaryPkgEbuildVisibilityTestCase(TestCase):
@@ -124,12 +129,24 @@ class BinaryPkgEbuildVisibilityTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- binpkgs=binpkgs, ebuilds=ebuilds, installed=installed, world=world
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ binpkgs=binpkgs,
+ ebuilds=ebuilds,
+ installed=installed,
+ world=world,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/test_changed_deps.py b/lib/portage/tests/resolver/test_changed_deps.py
index c3a0e2f87..0c9b34d4b 100644
--- a/lib/portage/tests/resolver/test_changed_deps.py
+++ b/lib/portage/tests/resolver/test_changed_deps.py
@@ -1,11 +1,16 @@
# Copyright 2014 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class ChangedDepsTestCase(TestCase):
@@ -103,16 +108,26 @@ class ChangedDepsTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- debug=False,
- ebuilds=ebuilds,
- binpkgs=binpkgs,
- installed=installed,
- world=world,
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ debug=False,
+ ebuilds=ebuilds,
+ binpkgs=binpkgs,
+ installed=installed,
+ world=world,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/test_complete_if_new_subslot_without_revbump.py b/lib/portage/tests/resolver/test_complete_if_new_subslot_without_revbump.py
index 3a5912606..6b061ca2a 100644
--- a/lib/portage/tests/resolver/test_complete_if_new_subslot_without_revbump.py
+++ b/lib/portage/tests/resolver/test_complete_if_new_subslot_without_revbump.py
@@ -1,11 +1,16 @@
# Copyright 2013 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class CompeteIfNewSubSlotWithoutRevBumpTestCase(TestCase):
@@ -55,16 +60,25 @@ class CompeteIfNewSubSlotWithoutRevBumpTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- ebuilds=ebuilds,
- binpkgs=binpkgs,
- installed=installed,
- world=world,
- debug=False,
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ ebuilds=ebuilds,
+ binpkgs=binpkgs,
+ installed=installed,
+ world=world,
+ debug=False,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/test_disjunctive_depend_order.py b/lib/portage/tests/resolver/test_disjunctive_depend_order.py
index e08a1d845..4471bc605 100644
--- a/lib/portage/tests/resolver/test_disjunctive_depend_order.py
+++ b/lib/portage/tests/resolver/test_disjunctive_depend_order.py
@@ -1,11 +1,16 @@
# Copyright 2017 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class DisjunctiveDependOrderTestCase(TestCase):
@@ -71,12 +76,25 @@ class DisjunctiveDependOrderTestCase(TestCase):
),
)
- playground = ResolverPlayground(debug=False, binpkgs=binpkgs, ebuilds=ebuilds)
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ debug=False,
+ binpkgs=binpkgs,
+ ebuilds=ebuilds,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.debug = False
- playground.cleanup()
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.debug = False
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/test_multirepo.py b/lib/portage/tests/resolver/test_multirepo.py
index 3a8eaa3d6..8132439b5 100644
--- a/lib/portage/tests/resolver/test_multirepo.py
+++ b/lib/portage/tests/resolver/test_multirepo.py
@@ -1,11 +1,16 @@
# Copyright 2010-2020 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class MultirepoTestCase(TestCase):
@@ -236,15 +241,28 @@ class MultirepoTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- ebuilds=ebuilds, binpkgs=binpkgs, installed=installed, sets=sets
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ ebuilds=ebuilds,
+ binpkgs=binpkgs,
+ installed=installed,
+ sets=sets,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.cleanup()
def testMultirepoUserConfig(self):
ebuilds = {
@@ -382,12 +400,20 @@ class MultirepoTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- ebuilds=ebuilds, installed=installed, user_config=user_config
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ user_config["make.conf"] = ('BINPKG_FORMAT="%s"' % binpkg_format,)
+ playground = ResolverPlayground(
+ ebuilds=ebuilds, installed=installed, user_config=user_config
+ )
+
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/test_regular_slot_change_without_revbump.py b/lib/portage/tests/resolver/test_regular_slot_change_without_revbump.py
index bb1ce0e87..127a175d1 100644
--- a/lib/portage/tests/resolver/test_regular_slot_change_without_revbump.py
+++ b/lib/portage/tests/resolver/test_regular_slot_change_without_revbump.py
@@ -1,11 +1,16 @@
# Copyright 2013 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class RegularSlotChangeWithoutRevBumpTestCase(TestCase):
@@ -42,16 +47,26 @@ class RegularSlotChangeWithoutRevBumpTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- ebuilds=ebuilds,
- binpkgs=binpkgs,
- installed=installed,
- world=world,
- debug=False,
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ ebuilds=ebuilds,
+ binpkgs=binpkgs,
+ installed=installed,
+ world=world,
+ debug=False,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/test_simple.py b/lib/portage/tests/resolver/test_simple.py
index 9bcf446be..b2bfd5fdf 100644
--- a/lib/portage/tests/resolver/test_simple.py
+++ b/lib/portage/tests/resolver/test_simple.py
@@ -1,11 +1,16 @@
# Copyright 2010-2020 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class SimpleResolverTestCase(TestCase):
@@ -75,12 +80,23 @@ class SimpleResolverTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- ebuilds=ebuilds, binpkgs=binpkgs, installed=installed
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ ebuilds=ebuilds,
+ binpkgs=binpkgs,
+ installed=installed,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/test_slot_abi.py b/lib/portage/tests/resolver/test_slot_abi.py
index afab001df..2d99fb676 100644
--- a/lib/portage/tests/resolver/test_slot_abi.py
+++ b/lib/portage/tests/resolver/test_slot_abi.py
@@ -1,11 +1,16 @@
# Copyright 2012-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class SlotAbiTestCase(TestCase):
@@ -118,19 +123,29 @@ class SlotAbiTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- ebuilds=ebuilds,
- binpkgs=binpkgs,
- installed=installed,
- world=world,
- debug=False,
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ ebuilds=ebuilds,
+ binpkgs=binpkgs,
+ installed=installed,
+ world=world,
+ debug=False,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.cleanup()
def testWholeSlot(self):
ebuilds = {
@@ -243,19 +258,29 @@ class SlotAbiTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- ebuilds=ebuilds,
- binpkgs=binpkgs,
- installed=installed,
- world=world,
- debug=False,
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ ebuilds=ebuilds,
+ binpkgs=binpkgs,
+ installed=installed,
+ world=world,
+ debug=False,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.cleanup()
def testWholeSlotConditional(self):
ebuilds = {
@@ -447,16 +472,26 @@ class SlotAbiTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- ebuilds=ebuilds,
- binpkgs=binpkgs,
- installed=installed,
- world=world,
- debug=False,
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ ebuilds=ebuilds,
+ binpkgs=binpkgs,
+ installed=installed,
+ world=world,
+ debug=False,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/test_slot_abi_downgrade.py b/lib/portage/tests/resolver/test_slot_abi_downgrade.py
index badd31b2d..15c6a7bfc 100644
--- a/lib/portage/tests/resolver/test_slot_abi_downgrade.py
+++ b/lib/portage/tests/resolver/test_slot_abi_downgrade.py
@@ -1,11 +1,16 @@
# Copyright 2012-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class SlotAbiDowngradeTestCase(TestCase):
@@ -96,19 +101,29 @@ class SlotAbiDowngradeTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- ebuilds=ebuilds,
- binpkgs=binpkgs,
- installed=installed,
- world=world,
- debug=False,
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ ebuilds=ebuilds,
+ binpkgs=binpkgs,
+ installed=installed,
+ world=world,
+ debug=False,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.cleanup()
def testWholeSlotSubSlotMix(self):
ebuilds = {
@@ -197,16 +212,26 @@ class SlotAbiDowngradeTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- ebuilds=ebuilds,
- binpkgs=binpkgs,
- installed=installed,
- world=world,
- debug=False,
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ ebuilds=ebuilds,
+ binpkgs=binpkgs,
+ installed=installed,
+ world=world,
+ debug=False,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/test_slot_change_without_revbump.py b/lib/portage/tests/resolver/test_slot_change_without_revbump.py
index 3dbd4f75e..c1c727caf 100644
--- a/lib/portage/tests/resolver/test_slot_change_without_revbump.py
+++ b/lib/portage/tests/resolver/test_slot_change_without_revbump.py
@@ -1,11 +1,16 @@
# Copyright 2013 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class SlotChangeWithoutRevBumpTestCase(TestCase):
@@ -71,16 +76,25 @@ class SlotChangeWithoutRevBumpTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- ebuilds=ebuilds,
- binpkgs=binpkgs,
- installed=installed,
- world=world,
- debug=False,
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ ebuilds=ebuilds,
+ binpkgs=binpkgs,
+ installed=installed,
+ world=world,
+ debug=False,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/test_slot_operator_autounmask.py b/lib/portage/tests/resolver/test_slot_operator_autounmask.py
index 7d6c3af26..b8b502a68 100644
--- a/lib/portage/tests/resolver/test_slot_operator_autounmask.py
+++ b/lib/portage/tests/resolver/test_slot_operator_autounmask.py
@@ -1,11 +1,16 @@
# Copyright 2013-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class SlotOperatorAutoUnmaskTestCase(TestCase):
@@ -109,16 +114,25 @@ class SlotOperatorAutoUnmaskTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- ebuilds=ebuilds,
- binpkgs=binpkgs,
- installed=installed,
- world=world,
- debug=False,
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ ebuilds=ebuilds,
+ binpkgs=binpkgs,
+ installed=installed,
+ world=world,
+ debug=False,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/test_slot_operator_bdeps.py b/lib/portage/tests/resolver/test_slot_operator_bdeps.py
index 0b1f426b7..6f0e5f7e1 100644
--- a/lib/portage/tests/resolver/test_slot_operator_bdeps.py
+++ b/lib/portage/tests/resolver/test_slot_operator_bdeps.py
@@ -1,11 +1,13 @@
# Copyright 2020 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class SlotOperatorBdependTestCase(TestCase):
@@ -88,20 +90,28 @@ class SlotOperatorBdependTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- ebuilds=ebuilds,
- binpkgs=binpkgs,
- installed=installed,
- world=world,
- debug=False,
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.debug = False
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ playground = ResolverPlayground(
+ ebuilds=ebuilds,
+ binpkgs=binpkgs,
+ installed=installed,
+ world=world,
+ debug=False,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.debug = False
+ playground.cleanup()
def testSlotOperatorBdependAfterBreakage(self):
"""
@@ -182,17 +192,25 @@ class SlotOperatorBdependTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- ebuilds=ebuilds,
- binpkgs=binpkgs,
- installed=installed,
- world=world,
- debug=False,
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.debug = False
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ playground = ResolverPlayground(
+ ebuilds=ebuilds,
+ binpkgs=binpkgs,
+ installed=installed,
+ world=world,
+ debug=False,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.debug = False
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/test_slot_operator_rebuild.py b/lib/portage/tests/resolver/test_slot_operator_rebuild.py
index 9e2325afb..9ad2bc7ab 100644
--- a/lib/portage/tests/resolver/test_slot_operator_rebuild.py
+++ b/lib/portage/tests/resolver/test_slot_operator_rebuild.py
@@ -1,11 +1,16 @@
# Copyright 2014-2018 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class SlotOperatorRebuildTestCase(TestCase):
@@ -72,16 +77,25 @@ class SlotOperatorRebuildTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- ebuilds=ebuilds,
- binpkgs=binpkgs,
- installed=installed,
- world=world,
- debug=False,
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ ebuilds=ebuilds,
+ binpkgs=binpkgs,
+ installed=installed,
+ world=world,
+ debug=False,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/test_slot_operator_unsolved.py b/lib/portage/tests/resolver/test_slot_operator_unsolved.py
index d43e8367d..945e34ccf 100644
--- a/lib/portage/tests/resolver/test_slot_operator_unsolved.py
+++ b/lib/portage/tests/resolver/test_slot_operator_unsolved.py
@@ -1,11 +1,16 @@
# Copyright 2013 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class SlotOperatorUnsolvedTestCase(TestCase):
@@ -71,17 +76,25 @@ class SlotOperatorUnsolvedTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- ebuilds=ebuilds,
- binpkgs=binpkgs,
- installed=installed,
- user_config=user_config,
- world=world,
- debug=False,
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ _user_config = user_config.copy()
+ _user_config["make.conf"] += ('BINPKG_FORMAT="%s"' % binpkg_format,)
+ playground = ResolverPlayground(
+ ebuilds=ebuilds,
+ binpkgs=binpkgs,
+ installed=installed,
+ user_config=_user_config,
+ world=world,
+ debug=False,
+ )
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/test_unecessary_slot_upgrade.py b/lib/portage/tests/resolver/test_unecessary_slot_upgrade.py
new file mode 100644
index 000000000..f8b8b346a
--- /dev/null
+++ b/lib/portage/tests/resolver/test_unecessary_slot_upgrade.py
@@ -0,0 +1,62 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from portage.tests import TestCase
+from portage.tests.resolver.ResolverPlayground import (
+ ResolverPlayground,
+ ResolverPlaygroundTestCase,
+)
+
+
+class UnnecessarySlotrUpgradeTestCase(TestCase):
+ def testUnnecessarySlotUpgrade(self):
+ ebuilds = {
+ "app-misc/a-1": {
+ "EAPI": "8",
+ "RDEPEND": "|| ( dev-lang/python:3.10 dev-lang/python:3.9 ) || ( dev-lang/python:3.10 dev-lang/python:3.9 )",
+ },
+ "dev-lang/python-3.9": {"SLOT": "3.9"},
+ "dev-lang/python-3.10": {"SLOT": "3.10"},
+ }
+
+ installed = {
+ "dev-lang/python-3.9": {"SLOT": "3.9"},
+ }
+
+ test_cases = (
+ # Test bug 828136, where an unnecessary python slot upgrade
+ # was triggered.
+ ResolverPlaygroundTestCase(
+ [
+ "app-misc/a",
+ ],
+ success=True,
+ mergelist=("app-misc/a-1",),
+ ),
+ ResolverPlaygroundTestCase(
+ [
+ "app-misc/a",
+ ],
+ success=True,
+ mergelist=(
+ "dev-lang/python-3.10",
+ "app-misc/a-1",
+ ),
+ options={
+ "--deep": True,
+ "--update": True,
+ },
+ ),
+ )
+
+ playground = ResolverPlayground(
+ debug=False, ebuilds=ebuilds, installed=installed
+ )
+
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(test_case.test_success, True, test_case.fail_msg)
+ finally:
+ playground.debug = False
+ playground.cleanup()
diff --git a/lib/portage/tests/resolver/test_useflags.py b/lib/portage/tests/resolver/test_useflags.py
index 6d74807e5..340ac5de2 100644
--- a/lib/portage/tests/resolver/test_useflags.py
+++ b/lib/portage/tests/resolver/test_useflags.py
@@ -1,11 +1,16 @@
# Copyright 2014 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
+
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import (
ResolverPlayground,
ResolverPlaygroundTestCase,
)
+from portage.output import colorize
class UseFlagsTestCase(TestCase):
@@ -118,15 +123,23 @@ class UseFlagsTestCase(TestCase):
),
)
- playground = ResolverPlayground(
- ebuilds=ebuilds,
- binpkgs=binpkgs,
- installed=installed,
- user_config=user_config,
- )
- try:
- for test_case in test_cases:
- playground.run_TestCase(test_case)
- self.assertEqual(test_case.test_success, True, test_case.fail_msg)
- finally:
- playground.cleanup()
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ user_config["make.conf"] = ('BINPKG_FORMAT="%s"' % binpkg_format,)
+ playground = ResolverPlayground(
+ ebuilds=ebuilds,
+ binpkgs=binpkgs,
+ installed=installed,
+ user_config=user_config,
+ )
+
+ try:
+ for test_case in test_cases:
+ playground.run_TestCase(test_case)
+ self.assertEqual(
+ test_case.test_success, True, test_case.fail_msg
+ )
+ finally:
+ playground.cleanup()
diff --git a/lib/portage/tests/runTests.py b/lib/portage/tests/runTests.py
index 85b746092..167b40d5a 100755
--- a/lib/portage/tests/runTests.py
+++ b/lib/portage/tests/runTests.py
@@ -9,7 +9,10 @@ import os.path as osp
import platform
import pwd
import signal
+import tempfile
+import shutil
import sys
+from distutils.dir_util import copy_tree
def debug_signal(signum, frame):
@@ -63,8 +66,17 @@ if insert_bin_path:
path.insert(0, PORTAGE_BIN_PATH)
os.environ["PATH"] = ":".join(path)
+# Copy GPG test keys to temporary directory
+gpg_path = tempfile.mkdtemp(prefix="gpg_")
+
+copy_tree(os.path.join(os.path.dirname(os.path.realpath(__file__)), ".gnupg"), gpg_path)
+
+os.chmod(gpg_path, 0o700)
+os.environ["PORTAGE_GNUPGHOME"] = gpg_path
+
if __name__ == "__main__":
try:
sys.exit(tests.main())
finally:
global_event_loop().close()
+ shutil.rmtree(gpg_path, ignore_errors=True)
diff --git a/lib/portage/tests/update/test_move_ent.py b/lib/portage/tests/update/test_move_ent.py
index ba5add989..cb9bb5243 100644
--- a/lib/portage/tests/update/test_move_ent.py
+++ b/lib/portage/tests/update/test_move_ent.py
@@ -1,14 +1,18 @@
# Copyright 2012-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
import textwrap
import portage
from portage import os
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import ResolverPlayground
from portage.util import ensure_dirs
from portage._global_updates import _do_global_updates
+from portage.output import colorize
class MoveEntTestCase(TestCase):
@@ -47,60 +51,73 @@ class MoveEntTestCase(TestCase):
"""
)
- playground = ResolverPlayground(
- binpkgs=binpkgs, ebuilds=ebuilds, installed=installed
- )
-
- settings = playground.settings
- trees = playground.trees
- eroot = settings["EROOT"]
- test_repo_location = settings.repositories["test_repo"].location
- portdb = trees[eroot]["porttree"].dbapi
- vardb = trees[eroot]["vartree"].dbapi
- bindb = trees[eroot]["bintree"].dbapi
-
- updates_dir = os.path.join(test_repo_location, "profiles", "updates")
-
- try:
- ensure_dirs(updates_dir)
- with open(os.path.join(updates_dir, "1Q-2010"), "w") as f:
- f.write(updates)
-
- # Create an empty updates directory, so that this
- # repo doesn't inherit updates from the main repo.
- ensure_dirs(
- os.path.join(
- portdb.getRepositoryPath("dont_apply_updates"),
- "profiles",
- "updates",
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ binpkgs=binpkgs,
+ ebuilds=ebuilds,
+ installed=installed,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
)
- )
-
- global_noiselimit = portage.util.noiselimit
- portage.util.noiselimit = -2
- try:
- _do_global_updates(trees, {})
- finally:
- portage.util.noiselimit = global_noiselimit
-
- # Workaround for cache validation not working
- # correctly when filesystem has timestamp precision
- # of 1 second.
- vardb._clear_cache()
-
- # A -> A-moved
- self.assertRaises(KeyError, vardb.aux_get, "dev-libs/A-1", ["EAPI"])
- vardb.aux_get("dev-libs/A-moved-1", ["EAPI"])
- # The original package should still exist because a binary
- # package move is a copy on write operation.
- bindb.aux_get("dev-libs/A-1", ["EAPI"])
- bindb.aux_get("dev-libs/A-moved-1", ["EAPI"])
-
- # dont_apply_updates
- self.assertRaises(KeyError, vardb.aux_get, "dev-libs/A-moved-2", ["EAPI"])
- vardb.aux_get("dev-libs/A-2", ["EAPI"])
- self.assertRaises(KeyError, bindb.aux_get, "dev-libs/A-moved-2", ["EAPI"])
- bindb.aux_get("dev-libs/A-2", ["EAPI"])
-
- finally:
- playground.cleanup()
+
+ settings = playground.settings
+ trees = playground.trees
+ eroot = settings["EROOT"]
+ test_repo_location = settings.repositories["test_repo"].location
+ portdb = trees[eroot]["porttree"].dbapi
+ vardb = trees[eroot]["vartree"].dbapi
+ bindb = trees[eroot]["bintree"].dbapi
+
+ updates_dir = os.path.join(test_repo_location, "profiles", "updates")
+
+ try:
+ ensure_dirs(updates_dir)
+ with open(os.path.join(updates_dir, "1Q-2010"), "w") as f:
+ f.write(updates)
+
+ # Create an empty updates directory, so that this
+ # repo doesn't inherit updates from the main repo.
+ ensure_dirs(
+ os.path.join(
+ portdb.getRepositoryPath("dont_apply_updates"),
+ "profiles",
+ "updates",
+ )
+ )
+
+ global_noiselimit = portage.util.noiselimit
+ portage.util.noiselimit = -2
+ try:
+ _do_global_updates(trees, {})
+ finally:
+ portage.util.noiselimit = global_noiselimit
+
+ # Workaround for cache validation not working
+ # correctly when filesystem has timestamp precision
+ # of 1 second.
+ vardb._clear_cache()
+
+ # A -> A-moved
+ self.assertRaises(KeyError, vardb.aux_get, "dev-libs/A-1", ["EAPI"])
+ vardb.aux_get("dev-libs/A-moved-1", ["EAPI"])
+ # The original package should still exist because a binary
+ # package move is a copy on write operation.
+ bindb.aux_get("dev-libs/A-1", ["EAPI"])
+ bindb.aux_get("dev-libs/A-moved-1", ["EAPI"])
+
+ # dont_apply_updates
+ self.assertRaises(
+ KeyError, vardb.aux_get, "dev-libs/A-moved-2", ["EAPI"]
+ )
+ vardb.aux_get("dev-libs/A-2", ["EAPI"])
+ self.assertRaises(
+ KeyError, bindb.aux_get, "dev-libs/A-moved-2", ["EAPI"]
+ )
+ bindb.aux_get("dev-libs/A-2", ["EAPI"])
+
+ finally:
+ playground.cleanup()
diff --git a/lib/portage/tests/update/test_move_slot_ent.py b/lib/portage/tests/update/test_move_slot_ent.py
index 87e38f97e..27d51fbb1 100644
--- a/lib/portage/tests/update/test_move_slot_ent.py
+++ b/lib/portage/tests/update/test_move_slot_ent.py
@@ -1,14 +1,18 @@
# Copyright 2012-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
import textwrap
import portage
from portage import os
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import ResolverPlayground
from portage.util import ensure_dirs
from portage._global_updates import _do_global_updates
+from portage.output import colorize
class MoveSlotEntTestCase(TestCase):
@@ -75,63 +79,80 @@ class MoveSlotEntTestCase(TestCase):
"""
)
- playground = ResolverPlayground(
- binpkgs=binpkgs, ebuilds=ebuilds, installed=installed
- )
-
- settings = playground.settings
- trees = playground.trees
- eroot = settings["EROOT"]
- test_repo_location = settings.repositories["test_repo"].location
- portdb = trees[eroot]["porttree"].dbapi
- vardb = trees[eroot]["vartree"].dbapi
- bindb = trees[eroot]["bintree"].dbapi
-
- updates_dir = os.path.join(test_repo_location, "profiles", "updates")
-
- try:
- ensure_dirs(updates_dir)
- with open(os.path.join(updates_dir, "1Q-2010"), "w") as f:
- f.write(updates)
-
- # Create an empty updates directory, so that this
- # repo doesn't inherit updates from the main repo.
- ensure_dirs(
- os.path.join(
- portdb.getRepositoryPath("dont_apply_updates"),
- "profiles",
- "updates",
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ binpkgs=binpkgs,
+ ebuilds=ebuilds,
+ installed=installed,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
)
- )
-
- global_noiselimit = portage.util.noiselimit
- portage.util.noiselimit = -2
- try:
- _do_global_updates(trees, {})
- finally:
- portage.util.noiselimit = global_noiselimit
-
- # Workaround for cache validation not working
- # correctly when filesystem has timestamp precision
- # of 1 second.
- vardb._clear_cache()
-
- # 0/2.30 -> 2/2.30
- self.assertEqual("2/2.30", vardb.aux_get("dev-libs/A-1", ["SLOT"])[0])
- self.assertEqual("2/2.30", bindb.aux_get("dev-libs/A-1", ["SLOT"])[0])
-
- # 0 -> 1
- self.assertEqual("1", vardb.aux_get("dev-libs/B-1", ["SLOT"])[0])
- self.assertEqual("1", bindb.aux_get("dev-libs/B-1", ["SLOT"])[0])
-
- # 0/1 -> 1 (equivalent to 1/1)
- self.assertEqual("1", vardb.aux_get("dev-libs/C-1", ["SLOT"])[0])
- self.assertEqual("1", bindb.aux_get("dev-libs/C-1", ["SLOT"])[0])
-
- # dont_apply_updates
- self.assertEqual("0/2.30", bindb.aux_get("dev-libs/A-2", ["SLOT"])[0])
- self.assertEqual("0", bindb.aux_get("dev-libs/B-2", ["SLOT"])[0])
- self.assertEqual("0/2.1", bindb.aux_get("dev-libs/C-2.1", ["SLOT"])[0])
-
- finally:
- playground.cleanup()
+
+ settings = playground.settings
+ trees = playground.trees
+ eroot = settings["EROOT"]
+ test_repo_location = settings.repositories["test_repo"].location
+ portdb = trees[eroot]["porttree"].dbapi
+ vardb = trees[eroot]["vartree"].dbapi
+ bindb = trees[eroot]["bintree"].dbapi
+
+ updates_dir = os.path.join(test_repo_location, "profiles", "updates")
+
+ try:
+ ensure_dirs(updates_dir)
+ with open(os.path.join(updates_dir, "1Q-2010"), "w") as f:
+ f.write(updates)
+
+ # Create an empty updates directory, so that this
+ # repo doesn't inherit updates from the main repo.
+ ensure_dirs(
+ os.path.join(
+ portdb.getRepositoryPath("dont_apply_updates"),
+ "profiles",
+ "updates",
+ )
+ )
+
+ global_noiselimit = portage.util.noiselimit
+ portage.util.noiselimit = -2
+ try:
+ _do_global_updates(trees, {})
+ finally:
+ portage.util.noiselimit = global_noiselimit
+
+ # Workaround for cache validation not working
+ # correctly when filesystem has timestamp precision
+ # of 1 second.
+ vardb._clear_cache()
+
+ # 0/2.30 -> 2/2.30
+ self.assertEqual(
+ "2/2.30", vardb.aux_get("dev-libs/A-1", ["SLOT"])[0]
+ )
+ self.assertEqual(
+ "2/2.30", bindb.aux_get("dev-libs/A-1", ["SLOT"])[0]
+ )
+
+ # 0 -> 1
+ self.assertEqual("1", vardb.aux_get("dev-libs/B-1", ["SLOT"])[0])
+ self.assertEqual("1", bindb.aux_get("dev-libs/B-1", ["SLOT"])[0])
+
+ # 0/1 -> 1 (equivalent to 1/1)
+ self.assertEqual("1", vardb.aux_get("dev-libs/C-1", ["SLOT"])[0])
+ self.assertEqual("1", bindb.aux_get("dev-libs/C-1", ["SLOT"])[0])
+
+ # dont_apply_updates
+ self.assertEqual(
+ "0/2.30", bindb.aux_get("dev-libs/A-2", ["SLOT"])[0]
+ )
+ self.assertEqual("0", bindb.aux_get("dev-libs/B-2", ["SLOT"])[0])
+ self.assertEqual(
+ "0/2.1", bindb.aux_get("dev-libs/C-2.1", ["SLOT"])[0]
+ )
+
+ finally:
+ playground.cleanup()
diff --git a/lib/portage/tests/update/test_update_dbentry.py b/lib/portage/tests/update/test_update_dbentry.py
index bed0f4b7c..3b49cba9d 100644
--- a/lib/portage/tests/update/test_update_dbentry.py
+++ b/lib/portage/tests/update/test_update_dbentry.py
@@ -1,11 +1,14 @@
# Copyright 2012-2013 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
+from __future__ import print_function
+import sys
import re
import textwrap
import portage
from portage import os
+from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS
from portage.dep import Atom
from portage.tests import TestCase
from portage.tests.resolver.ResolverPlayground import ResolverPlayground
@@ -13,6 +16,7 @@ from portage.update import update_dbentry
from portage.util import ensure_dirs
from portage.versions import _pkg_str
from portage._global_updates import _do_global_updates
+from portage.output import colorize
class UpdateDbentryTestCase(TestCase):
@@ -224,98 +228,110 @@ class UpdateDbentryTestCase(TestCase):
"""
)
- playground = ResolverPlayground(
- binpkgs=binpkgs, ebuilds=ebuilds, installed=installed, world=world
- )
+ for binpkg_format in SUPPORTED_GENTOO_BINPKG_FORMATS:
+ with self.subTest(binpkg_format=binpkg_format):
+ print(colorize("HILITE", binpkg_format), end=" ... ")
+ sys.stdout.flush()
+ playground = ResolverPlayground(
+ binpkgs=binpkgs,
+ ebuilds=ebuilds,
+ installed=installed,
+ world=world,
+ user_config={
+ "make.conf": ('BINPKG_FORMAT="%s"' % binpkg_format,),
+ },
+ )
- settings = playground.settings
- trees = playground.trees
- eroot = settings["EROOT"]
- test_repo_location = settings.repositories["test_repo"].location
- portdb = trees[eroot]["porttree"].dbapi
- vardb = trees[eroot]["vartree"].dbapi
- bindb = trees[eroot]["bintree"].dbapi
- setconfig = trees[eroot]["root_config"].setconfig
- selected_set = setconfig.getSets()["selected"]
+ settings = playground.settings
+ trees = playground.trees
+ eroot = settings["EROOT"]
+ test_repo_location = settings.repositories["test_repo"].location
+ portdb = trees[eroot]["porttree"].dbapi
+ vardb = trees[eroot]["vartree"].dbapi
+ bindb = trees[eroot]["bintree"].dbapi
+ setconfig = trees[eroot]["root_config"].setconfig
+ selected_set = setconfig.getSets()["selected"]
- updates_dir = os.path.join(test_repo_location, "profiles", "updates")
+ updates_dir = os.path.join(test_repo_location, "profiles", "updates")
- try:
- ensure_dirs(updates_dir)
- with open(os.path.join(updates_dir, "1Q-2010"), "w") as f:
- f.write(updates)
+ try:
+ ensure_dirs(updates_dir)
+ with open(os.path.join(updates_dir, "1Q-2010"), "w") as f:
+ f.write(updates)
- # Create an empty updates directory, so that this
- # repo doesn't inherit updates from the main repo.
- ensure_dirs(
- os.path.join(
- portdb.getRepositoryPath("dont_apply_updates"),
- "profiles",
- "updates",
- )
- )
+ # Create an empty updates directory, so that this
+ # repo doesn't inherit updates from the main repo.
+ ensure_dirs(
+ os.path.join(
+ portdb.getRepositoryPath("dont_apply_updates"),
+ "profiles",
+ "updates",
+ )
+ )
- global_noiselimit = portage.util.noiselimit
- portage.util.noiselimit = -2
- try:
- _do_global_updates(trees, {})
- finally:
- portage.util.noiselimit = global_noiselimit
+ global_noiselimit = portage.util.noiselimit
+ portage.util.noiselimit = -2
+ try:
+ _do_global_updates(trees, {})
+ finally:
+ portage.util.noiselimit = global_noiselimit
- # Workaround for cache validation not working
- # correctly when filesystem has timestamp precision
- # of 1 second.
- vardb._clear_cache()
+ # Workaround for cache validation not working
+ # correctly when filesystem has timestamp precision
+ # of 1 second.
+ vardb._clear_cache()
- # M -> M-moved
- old_pattern = re.compile(r"\bdev-libs/M(\s|$)")
- rdepend = vardb.aux_get("dev-libs/A-1", ["RDEPEND"])[0]
- self.assertTrue(old_pattern.search(rdepend) is None)
- self.assertTrue("dev-libs/M-moved" in rdepend)
- rdepend = bindb.aux_get("dev-libs/A-1", ["RDEPEND"])[0]
- self.assertTrue(old_pattern.search(rdepend) is None)
- self.assertTrue("dev-libs/M-moved" in rdepend)
- rdepend = vardb.aux_get("dev-libs/B-1", ["RDEPEND"])[0]
- self.assertTrue(old_pattern.search(rdepend) is None)
- self.assertTrue("dev-libs/M-moved" in rdepend)
- rdepend = vardb.aux_get("dev-libs/B-1", ["RDEPEND"])[0]
- self.assertTrue(old_pattern.search(rdepend) is None)
- self.assertTrue("dev-libs/M-moved" in rdepend)
+ # M -> M-moved
+ old_pattern = re.compile(r"\bdev-libs/M(\s|$)")
+ rdepend = vardb.aux_get("dev-libs/A-1", ["RDEPEND"])[0]
+ self.assertTrue(old_pattern.search(rdepend) is None)
+ self.assertTrue("dev-libs/M-moved" in rdepend)
+ rdepend = bindb.aux_get("dev-libs/A-1", ["RDEPEND"])[0]
+ self.assertTrue(old_pattern.search(rdepend) is None)
+ self.assertTrue("dev-libs/M-moved" in rdepend)
+ rdepend = vardb.aux_get("dev-libs/B-1", ["RDEPEND"])[0]
+ self.assertTrue(old_pattern.search(rdepend) is None)
+ self.assertTrue("dev-libs/M-moved" in rdepend)
+ rdepend = vardb.aux_get("dev-libs/B-1", ["RDEPEND"])[0]
+ self.assertTrue(old_pattern.search(rdepend) is None)
+ self.assertTrue("dev-libs/M-moved" in rdepend)
- # EAPI 4-python/*-progress N -> N.moved
- rdepend = vardb.aux_get("dev-libs/B-1", ["RDEPEND"])[0]
- old_pattern = re.compile(r"\bdev-libs/N(\s|$)")
- self.assertTrue(old_pattern.search(rdepend) is None)
- self.assertTrue("dev-libs/N.moved" in rdepend)
- rdepend = bindb.aux_get("dev-libs/B-1", ["RDEPEND"])[0]
- self.assertTrue(old_pattern.search(rdepend) is None)
- self.assertTrue("dev-libs/N.moved" in rdepend)
- self.assertRaises(KeyError, vardb.aux_get, "dev-libs/N-2", ["EAPI"])
- vardb.aux_get("dev-libs/N.moved-2", ["RDEPEND"])[0]
+ # EAPI 4-python/*-progress N -> N.moved
+ rdepend = vardb.aux_get("dev-libs/B-1", ["RDEPEND"])[0]
+ old_pattern = re.compile(r"\bdev-libs/N(\s|$)")
+ self.assertTrue(old_pattern.search(rdepend) is None)
+ self.assertTrue("dev-libs/N.moved" in rdepend)
+ rdepend = bindb.aux_get("dev-libs/B-1", ["RDEPEND"])[0]
+ self.assertTrue(old_pattern.search(rdepend) is None)
+ self.assertTrue("dev-libs/N.moved" in rdepend)
+ self.assertRaises(KeyError, vardb.aux_get, "dev-libs/N-2", ["EAPI"])
+ vardb.aux_get("dev-libs/N.moved-2", ["RDEPEND"])[0]
- # EAPI 4 does not allow dots in package names for N -> N.moved
- rdepend = vardb.aux_get("dev-libs/A-1", ["RDEPEND"])[0]
- self.assertTrue("dev-libs/N" in rdepend)
- self.assertTrue("dev-libs/N.moved" not in rdepend)
- rdepend = bindb.aux_get("dev-libs/A-1", ["RDEPEND"])[0]
- self.assertTrue("dev-libs/N" in rdepend)
- self.assertTrue("dev-libs/N.moved" not in rdepend)
- vardb.aux_get("dev-libs/N-1", ["RDEPEND"])[0]
- self.assertRaises(KeyError, vardb.aux_get, "dev-libs/N.moved-1", ["EAPI"])
+ # EAPI 4 does not allow dots in package names for N -> N.moved
+ rdepend = vardb.aux_get("dev-libs/A-1", ["RDEPEND"])[0]
+ self.assertTrue("dev-libs/N" in rdepend)
+ self.assertTrue("dev-libs/N.moved" not in rdepend)
+ rdepend = bindb.aux_get("dev-libs/A-1", ["RDEPEND"])[0]
+ self.assertTrue("dev-libs/N" in rdepend)
+ self.assertTrue("dev-libs/N.moved" not in rdepend)
+ vardb.aux_get("dev-libs/N-1", ["RDEPEND"])[0]
+ self.assertRaises(
+ KeyError, vardb.aux_get, "dev-libs/N.moved-1", ["EAPI"]
+ )
- # dont_apply_updates
- rdepend = vardb.aux_get("dev-libs/A-2", ["RDEPEND"])[0]
- self.assertTrue("dev-libs/M" in rdepend)
- self.assertTrue("dev-libs/M-moved" not in rdepend)
- rdepend = bindb.aux_get("dev-libs/A-2", ["RDEPEND"])[0]
- self.assertTrue("dev-libs/M" in rdepend)
- self.assertTrue("dev-libs/M-moved" not in rdepend)
+ # dont_apply_updates
+ rdepend = vardb.aux_get("dev-libs/A-2", ["RDEPEND"])[0]
+ self.assertTrue("dev-libs/M" in rdepend)
+ self.assertTrue("dev-libs/M-moved" not in rdepend)
+ rdepend = bindb.aux_get("dev-libs/A-2", ["RDEPEND"])[0]
+ self.assertTrue("dev-libs/M" in rdepend)
+ self.assertTrue("dev-libs/M-moved" not in rdepend)
- selected_set.load()
- self.assertTrue("dev-libs/M" not in selected_set)
- self.assertTrue("dev-libs/M-moved" in selected_set)
- self.assertTrue("dev-libs/N" not in selected_set)
- self.assertTrue("dev-libs/N.moved" in selected_set)
+ selected_set.load()
+ self.assertTrue("dev-libs/M" not in selected_set)
+ self.assertTrue("dev-libs/M-moved" in selected_set)
+ self.assertTrue("dev-libs/N" not in selected_set)
+ self.assertTrue("dev-libs/N.moved" in selected_set)
- finally:
- playground.cleanup()
+ finally:
+ playground.cleanup()
diff --git a/lib/portage/util/_dyn_libs/dyn_libs.py b/lib/portage/util/_dyn_libs/dyn_libs.py
new file mode 100644
index 000000000..ee28e8839
--- /dev/null
+++ b/lib/portage/util/_dyn_libs/dyn_libs.py
@@ -0,0 +1,28 @@
+# Copyright 2021 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+import os
+
+
+def installed_dynlibs(directory):
+ for _dirpath, _dirnames, filenames in os.walk(directory):
+ for filename in filenames:
+ if filename.endswith(".so"):
+ return True
+ return False
+
+
+def check_dyn_libs_inconsistent(directory, provides):
+ """Checks directory for whether any dynamic libraries were installed and
+ if PROVIDES corresponds."""
+
+ # Let's check if we've got inconsistent results.
+ # If we're installing dynamic libraries (.so files), we should
+ # really have a PROVIDES.
+ # (This is a complementary check at the point of ingestion for the
+ # creation check in doebuild.py)
+ # Note: we could check a non-empty PROVIDES against the list of .sos,
+ # but this doesn't gain us anything. We're interested in failure
+ # to properly parse the installed files at all, which should really
+ # be a global problem (e.g. bug #811462)
+ return not provides and installed_dynlibs(directory)
diff --git a/lib/portage/util/_urlopen.py b/lib/portage/util/_urlopen.py
index 70440c3e1..22f0e08df 100644
--- a/lib/portage/util/_urlopen.py
+++ b/lib/portage/util/_urlopen.py
@@ -26,7 +26,7 @@ def have_pep_476():
return hasattr(__import__("ssl"), "_create_unverified_context")
-def urlopen(url, if_modified_since=None, proxies=None):
+def urlopen(url, if_modified_since=None, headers={}, proxies=None):
parse_result = urllib_parse.urlparse(url)
if parse_result.scheme not in ("http", "https"):
return _urlopen(url)
@@ -45,6 +45,8 @@ def urlopen(url, if_modified_since=None, proxies=None):
password_manager = urllib_request.HTTPPasswordMgrWithDefaultRealm()
request = urllib_request.Request(url)
request.add_header("User-Agent", "Gentoo Portage")
+ for key in headers:
+ request.add_header(key, headers[key])
if if_modified_since:
request.add_header("If-Modified-Since", _timestamp_to_http(if_modified_since))
if parse_result.username is not None:
diff --git a/lib/portage/util/_xattr.py b/lib/portage/util/_xattr.py
index ff0a7d8c5..f4d3ad733 100644
--- a/lib/portage/util/_xattr.py
+++ b/lib/portage/util/_xattr.py
@@ -169,7 +169,6 @@ if hasattr(os, "getxattr"):
def list(item, nofollow=False, namespace=None):
return os.listxattr(item, follow_symlinks=not nofollow)
-
else:
try:
# Maybe we have the xattr module.
diff --git a/lib/portage/util/elf/constants.py b/lib/portage/util/elf/constants.py
index d86b39483..19b72cfa1 100644
--- a/lib/portage/util/elf/constants.py
+++ b/lib/portage/util/elf/constants.py
@@ -36,6 +36,7 @@ EM_X86_64 = 62
EM_ALTERA_NIOS2 = 113
EM_AARCH64 = 183
EM_RISCV = 243
+EM_LOONGARCH = 258
EM_ALPHA = 0x9026
E_ENTRY = 24
@@ -52,3 +53,11 @@ EF_RISCV_FLOAT_ABI_SOFT = 0x0000
EF_RISCV_FLOAT_ABI_SINGLE = 0x0002
EF_RISCV_FLOAT_ABI_DOUBLE = 0x0004
EF_RISCV_FLOAT_ABI_QUAD = 0x0006
+
+EF_LOONGARCH_ABI_LP64_SOFT_FLOAT = 0b001
+EF_LOONGARCH_ABI_LP64_SINGLE_FLOAT = 0b010
+EF_LOONGARCH_ABI_LP64_DOUBLE_FLOAT = 0b011
+EF_LOONGARCH_ABI_ILP32_SOFT_FLOAT = 0b101
+EF_LOONGARCH_ABI_ILP32_SINGLE_FLOAT = 0b110
+EF_LOONGARCH_ABI_ILP32_DOUBLE_FLOAT = 0b111
+EF_LOONGARCH_ABI_MASK = 0x07
diff --git a/lib/portage/util/futures/unix_events.py b/lib/portage/util/futures/unix_events.py
index f2f01f0c6..c9855aefb 100644
--- a/lib/portage/util/futures/unix_events.py
+++ b/lib/portage/util/futures/unix_events.py
@@ -23,7 +23,6 @@ if hasattr(os, "set_blocking"):
def _set_nonblocking(fd):
os.set_blocking(fd, False)
-
else:
def _set_nonblocking(fd):
diff --git a/lib/portage/versions.py b/lib/portage/versions.py
index fe1ff6ce0..086ada555 100644
--- a/lib/portage/versions.py
+++ b/lib/portage/versions.py
@@ -529,6 +529,22 @@ class _pkg_str(str):
self.__dict__["_stable"] = stable
return stable
+ @property
+ def binpkg_format(self):
+ """
+ Returns the BINPKG_FORMAT metadata. A return value of None means
+ that the format is unset. If there is no metadata available or the
+ BINPKG_FORMAT key is missing from the metadata, then raise
+ AttributeError.
+
+ @rtype: str or None
+ @return: a non-empty BINPKG_FORMAT string, or None
+ """
+ try:
+ return self._metadata["BINPKG_FORMAT"] or None
+ except (AttributeError, KeyError):
+ raise AttributeError("binpkg_format")
+
def pkgsplit(mypkg, silent=1, eapi=None):
"""
diff --git a/man/color.map.5 b/man/color.map.5
index 288bf7fbd..92a1baa91 100644
--- a/man/color.map.5
+++ b/man/color.map.5
@@ -40,7 +40,7 @@ Defines color used for numbers indicating merge progress.
\fBPKG_BLOCKER\fR = \fI"red"\fR
Defines color used for unsatisfied blockers.
.TP
-\fBPKG_BLOCKER_SATISFIED\fR = \fI"darkblue"\fR
+\fBPKG_BLOCKER_SATISFIED\fR = \fI"teal"\fR
Defines color used for satisfied blockers.
.TP
\fBPKG_MERGE\fR = \fI"darkgreen"\fR
@@ -63,10 +63,10 @@ package.
Defines color used for world packages planned to be merged using a binary
package.
.TP
-\fBPKG_NOMERGE\fR = \fI"darkblue"\fR
+\fBPKG_NOMERGE\fR = \fI"teal"\fR
Defines color used for packages not planned to be merged.
.TP
-\fBPKG_NOMERGE_SYSTEM\fR = \fI"darkblue"\fR
+\fBPKG_NOMERGE_SYSTEM\fR = \fI"teal"\fR
Defines color used for system packages not planned to be merged.
.TP
\fBPKG_NOMERGE_WORLD\fR = \fI"blue"\fR
diff --git a/man/emerge.1 b/man/emerge.1
index 8f8c2ebda..5ba88f3bc 100644
--- a/man/emerge.1
+++ b/man/emerge.1
@@ -1,4 +1,4 @@
-.TH "EMERGE" "1" "Mar 2021" "Portage VERSION" "Portage"
+.TH "EMERGE" "1" "Dec 2021" "Portage VERSION" "Portage"
.SH "NAME"
emerge \- Command\-line interface to the Portage system
.SH "SYNOPSIS"
@@ -588,7 +588,7 @@ dependency tree, as though no packages are currently
installed. You should run this with \fB\-\-pretend\fR
first to make sure the result is what you expect.
.TP
-.BR "\-\-exclude " ATOMS
+.BR "\-\-exclude, \-X ATOMS"
A space separated list of package names or slot atoms.
Emerge won't install any ebuild or binary package that
matches any of the given package atoms.
@@ -1064,6 +1064,9 @@ packages must be available at the time of dependency calculation or emerge
will simply abort. Portage does not use ebuild repositories when calculating
dependency information so all masking information is ignored.
.TP
+.BR "\-\-usepkg\-exclude\-live [ y | n ]"
+Tells emerge to not install from binary packages for live ebuilds.
+.TP
.BR "\-\-verbose [ y | n ]" ", " \-v
Tell emerge to run in verbose mode. Currently this flag causes emerge to print
out GNU info errors, if any, and to show the USE flags that will be used for
diff --git a/man/make.conf.5 b/man/make.conf.5
index 43e70803f..8afde4a00 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -1,4 +1,4 @@
-.TH "MAKE.CONF" "5" "Jun 2021" "Portage VERSION" "Portage"
+.TH "MAKE.CONF" "5" "Nov 2021" "Portage VERSION" "Portage"
.SH "NAME"
make.conf \- custom settings for Portage
.SH "SYNOPSIS"
@@ -137,6 +137,55 @@ Defaults to "".
BINPKG_COMPRESS_FLAGS="-9"
.fi
.TP
+\fBBINPKG_GPG_SIGNING_BASE_COMMAND\fR = \fI"GPG command and arguments \
+[PORTAGE_CONFIG]"\fR
+The base command will be used for all signing operations.
+Portage will replace \fB[PORTAGE_CONFIG]\fR under different operations.
+Please do not add arguments that can be configured independently.
+.br
+Defaults to "/usr/bin/flock /run/lock/portage-binpkg-gpg.lock /usr/bin/gpg \
+--sign --armor [PORTAGE_CONFIG]".
+.br
+.TP
+\fBBINPKG_GPG_SIGNING_DIGEST=\fR = \fI"GPG supported digest"\fR
+The digest that will be used for signature.
+.br
+Defaults to "SHA512"
+.br
+.TP
+\fBBINPKG_GPG_SIGNING_GPG_HOME\fR = \fI[path]\fR
+The GPG home where the signing private key located.
+.br
+Defaults to "/root/.gnupg"
+.br
+.TP
+\fBBINPKG_GPG_SIGNING_KEY\fR = \fI"GPG key ID"\fR
+GPG key ID used to sign binary packages, must exists in \
+\fBBINPKG_GPG_SIGNING_GPG_HOME\fR.
+.br
+Defaults to ""
+.br
+Example: "0x40DCF18E97150795!"
+.br
+.TP
+\fBBINPKG_GPG_VERIFY_BASE_COMMAND=\fR = \fI"GPG command and arguments"\fR
+The base command will be used for all verify operations.
+Portage will replace \fB[PORTAGE_CONFIG]\fR and \fB[SIGNATURE]\fR under \
+different operations.
+Please do not add arguments that can be configured independently.
+.br
+Defaults to "/usr/bin/gpg --verify --batch --no-tty --no-auto-check-trustdb \
+--status-fd 2 [PORTAGE_CONFIG] [SIGNATURE]"
+.br
+.TP
+\fBBINPKG_GPG_VERIFY_GPG_HOME\fR = \fI[path]\fR
+The GPG home where the trusted keys located. Please make sure the target \
+directory is globally readable, as the user will be dropped to \fBnobody\fR \
+during verification.
+.br
+Defaults to "/etc/portage/gnupg"
+.br
+.TP
.B CBUILD
This variable is passed by the \fIebuild scripts\fR to the \fIconfigure\fR
as \fI\-\-build=${CBUILD}\fR only if it is defined. Do not set this yourself
@@ -286,6 +335,11 @@ strips (or splits) them before installing.
\fBbinpkg\-dostrip\fR must be enabled for \fBinstallsources\fR to work.
.TP
+.B binpkg-ignore-signature
+This will disable GPG signature check for all binary packages. Enable this
+could be dangerous if you get binary packages from remote site or use third
+party packages.
+.TP
.B binpkg\-logs
Keep logs from successful binary package merges. This is relevant only when
\fBPORTAGE_LOGDIR\fR is set.
@@ -318,11 +372,26 @@ It is also possible to remove packages manually, and then run
\(aqemaint \-\-fix binhost' to update the ${PKGDIR}/Packages index.
This feature is enabled by default.
.TP
+.B binpkg-request-signature
+Binary packages are requested to be signed by trusted GPG signature.
+Portage will reject to process any binary package without a valid GPG
+signature. The verify command is defined in
+\fBBINPKG_GPG_VERIFY_COMMAND\fR variable.
+.TP
+.B binpkg-signing
+Binary packages will be signed by given GPG command. The signing command
+is defined in \fBBINPKG_GPG_SIGNING_COMMAND\fR variable.
+.TP
.B buildpkg
Binary packages will be created for all packages that are merged. Also see
\fBquickpkg\fR(1) and \fBemerge\fR(1) \fB\-\-buildpkg\fR and
\fB\-\-buildpkgonly\fR options.
.TP
+.B buildpkg\-live
+When this option is enabled (the default), \fBbuildpkg\fR will exhibit
+the default behavior of building binary cache for all packages. When
+it is disabled, binary packages will not be created for live ebuilds.
+.TP
.B buildsyspkg
Build binary packages for just packages in the system set.
.TP
@@ -446,6 +515,10 @@ for all EAPIs (for obvious reasons).
Force emerges to always try to fetch files from the \fIPORTAGE_BINHOST\fR. See
\fBmake.conf\fR(5) for more information.
.TP
+.B gpg-keepalive
+Run GPG unlock command every 5 mins to avoid the passphrase expired.
+If your GPG is auto unlocked on login, you do not need this.
+.TP
.B icecream
Enable portage support for the icecream package.
.TP
@@ -776,6 +849,18 @@ the \fIebuild scripts\fR. Merging 'mirrorselect' can help. Entries in this
variable that have no protocol and simply start with a '/' path separator may
be used to specify mounted filesystem mirrors.
.TP
+\fBGPG_VERIFY_GROUP_DROP\fR = \fI[group]\fR
+The group name used to drop root privileges during verification.
+.br
+Defaults to "nogroup"
+.br
+.TP
+\fBGPG_VERIFY_USER_DROP\fR = \fI[user]\fR
+The user name used to drop root privileges during verification.
+.br
+Defaults to "nobody"
+.br
+.TP
\fBhttp_proxy ftp_proxy RSYNC_PROXY\fR = \fI[protocol://host:port]\fR
These variables are used by network clients such as \fBwget\fR(1) and
\fBrsync\fR(1). They are only required if you use a
diff --git a/repoman/lib/repoman/tests/.gnupg/openpgp-revocs.d/06B3A311BD775C280D22A9305D90EA06352177F6.rev b/repoman/lib/repoman/tests/.gnupg/openpgp-revocs.d/06B3A311BD775C280D22A9305D90EA06352177F6.rev
new file mode 100644
index 000000000..a6752fd30
--- /dev/null
+++ b/repoman/lib/repoman/tests/.gnupg/openpgp-revocs.d/06B3A311BD775C280D22A9305D90EA06352177F6.rev
@@ -0,0 +1,37 @@
+This is a revocation certificate for the OpenPGP key:
+
+pub rsa4096 2020-07-14 [S]
+ 06B3A311BD775C280D22A9305D90EA06352177F6
+uid Gentoo Portage Test Trusted Key (Test Only, Do NOT Trust!!!) (Gentoo Test Key) <test@example.org>
+
+A revocation certificate is a kind of "kill switch" to publicly
+declare that a key shall not anymore be used. It is not possible
+to retract such a revocation certificate once it has been published.
+
+Use it to revoke this key in case of a compromise or loss of
+the secret key. However, if the secret key is still accessible,
+it is better to generate a new revocation certificate and give
+a reason for the revocation. For details see the description of
+of the gpg command "--generate-revocation" in the GnuPG manual.
+
+To avoid an accidental use of this file, a colon has been inserted
+before the 5 dashes below. Remove this colon with a text editor
+before importing and publishing this revocation certificate.
+
+:-----BEGIN PGP PUBLIC KEY BLOCK-----
+Comment: This is a revocation certificate
+
+iQI2BCABCAAgFiEEBrOjEb13XCgNIqkwXZDqBjUhd/YFAl8OFTwCHQAACgkQXZDq
+BjUhd/aXCA/+OgzosMDaDe5DNwkSi2yKdC2X18v8JcaYnXBUR93nXA0LVN7iVWkR
+WEH3NuVspQZ5vK+3AHTKabqZFC/buA5oQOH01Ncd4lQISfOOhFiBn5DIPX31BVT0
+iPmVkcxHAD4031ptP4oat6EFclT13SRchtlnAO04JofeHnzQIw3SozQGzXpAA1g4
+BogQ0HWA88HzuEYYE+e/yzZL4D496X1DTaXksg0Py5c4SS6u5pND6lcUtAGxAwa9
+sJFPs+coeURaRV99CrJfdh4u2OkvINTfrKOS6NFBQq6HVH5mLsRXZlcE4Oo4d+fN
+XoPrTZnRUqpJADUdjHFvO/lr0fArJTS5IQCVBNFeCMlvgmUPeKWJ1r6Uiwe/UHor
+9OP/tK97EqpsaXmHbo0jOUkn5iiUwy784+JBSSu/Q2NxqcBr74aaRdfxvs62dmv7
+droCDQi3ebqTdnlDSaeCIWHyVlSroOhZ+ZETVy193K1X7VXFX3hYKiJ3G8QZwy3e
+AlsVGjIHWfC+K+enIn+uwSUvOWPN3upK8kqMRuXvAOppFCE4sTqNbxUnHHXaqo/r
+s1q6zVsWVILBk97BHlJph2IaqhV7iIgPU97/r4U/BT11VqDFdVSHcXcs4PDNs5vh
+6qttaDiyDqZjwMr+0iDoouHxFpqY8e+3M2gycUgGr2XV6ML0pXE6BqA=
+=nIjC
+-----END PGP PUBLIC KEY BLOCK-----
diff --git a/repoman/lib/repoman/tests/.gnupg/openpgp-revocs.d/8DEDA2CDED49C8809287B89D8812797DDF1DD192.rev b/repoman/lib/repoman/tests/.gnupg/openpgp-revocs.d/8DEDA2CDED49C8809287B89D8812797DDF1DD192.rev
new file mode 100644
index 000000000..456e0aa50
--- /dev/null
+++ b/repoman/lib/repoman/tests/.gnupg/openpgp-revocs.d/8DEDA2CDED49C8809287B89D8812797DDF1DD192.rev
@@ -0,0 +1,37 @@
+This is a revocation certificate for the OpenPGP key:
+
+pub rsa4096 2020-07-14 [S]
+ 8DEDA2CDED49C8809287B89D8812797DDF1DD192
+uid Gentoo Portage Test Untrusted Key (Test Only, Do NOT Trust!!!) (Gentoo Test Key) <test@example.org>
+
+A revocation certificate is a kind of "kill switch" to publicly
+declare that a key shall not anymore be used. It is not possible
+to retract such a revocation certificate once it has been published.
+
+Use it to revoke this key in case of a compromise or loss of
+the secret key. However, if the secret key is still accessible,
+it is better to generate a new revocation certificate and give
+a reason for the revocation. For details see the description of
+of the gpg command "--generate-revocation" in the GnuPG manual.
+
+To avoid an accidental use of this file, a colon has been inserted
+before the 5 dashes below. Remove this colon with a text editor
+before importing and publishing this revocation certificate.
+
+:-----BEGIN PGP PUBLIC KEY BLOCK-----
+Comment: This is a revocation certificate
+
+iQI2BCABCAAgFiEEje2ize1JyICSh7idiBJ5fd8d0ZIFAl8OFXUCHQAACgkQiBJ5
+fd8d0ZKdwxAAhmkC0V+OLyOU9PCV6ogD9/3b3nVqNIreoc+gxHTLmEvxiMSItqmq
+DkcW9RJKAduA/HiLZQ8Yzxw+ldC6kuWqYEjNpSM54VDkrgOePi8W1bVDTCoSp7bo
+0JOG4frieqIxA6lhAA2UppH7EPRXoODPLYqooNxWAs3xxVrR6eGAb5l8NXzrymvN
+acFfOZ0s5FgADQskQHWVq6TaJn9DrcZxd+b+plSwPYDXqzTChKQ5jw7uMAPUvDkG
+JUWgoKiKSrK64bslUq8aEDEZQ4uxjyEi6G0vO/wPL/ysGhS7KkPgCZsEfNjWjajb
+jAsdvl1raoHxK/O7llMNr9uRAZtC56pJ//SRDc3kylZrkAo0RNoXQFowT739HWei
+2UkCFDfz488VKKrOI8TzTyUvLFEo14ZAXGg1wdHaGnbYMzxpKjP15alOFo6fKIcS
+Kz1f/Mab4wf4Sg0XAjQ9pnai1/U9ZF3/NSnRtYgJkLCrIEtRLrgSHJsLDPxjCfGV
+jWszAbIk167aA0yKsSmuwkpc5bZqqBaTo904r857fxyt5Les6SOHsV7iNXt7F+am
+03Y6u6m2eROba7M67l115vTyYcw5EZVp5j0nI81PXsC9X2DD1ci5xrNmPyEeupC4
+7y7mcGbUYPJAJHJ0kHG4ZYLnNMl42ZYr1ssEeasDwUsLWgVqvx9RkKI=
+=kVUQ
+-----END PGP PUBLIC KEY BLOCK-----
diff --git a/repoman/lib/repoman/tests/.gnupg/private-keys-v1.d/273B030399E7BEA66A9AD42216DE7CA17BA5D42E.key b/repoman/lib/repoman/tests/.gnupg/private-keys-v1.d/273B030399E7BEA66A9AD42216DE7CA17BA5D42E.key
new file mode 100644
index 000000000..0bd1026ad
--- /dev/null
+++ b/repoman/lib/repoman/tests/.gnupg/private-keys-v1.d/273B030399E7BEA66A9AD42216DE7CA17BA5D42E.key
Binary files differ
diff --git a/repoman/lib/repoman/tests/.gnupg/private-keys-v1.d/C99796FB85B0C3DF03314A11B5850C51167D6282.key b/repoman/lib/repoman/tests/.gnupg/private-keys-v1.d/C99796FB85B0C3DF03314A11B5850C51167D6282.key
new file mode 100644
index 000000000..8e29ef43c
--- /dev/null
+++ b/repoman/lib/repoman/tests/.gnupg/private-keys-v1.d/C99796FB85B0C3DF03314A11B5850C51167D6282.key
Binary files differ
diff --git a/repoman/lib/repoman/tests/.gnupg/pubring.kbx b/repoman/lib/repoman/tests/.gnupg/pubring.kbx
new file mode 100644
index 000000000..f6367f83b
--- /dev/null
+++ b/repoman/lib/repoman/tests/.gnupg/pubring.kbx
Binary files differ
diff --git a/repoman/lib/repoman/tests/.gnupg/trustdb.gpg b/repoman/lib/repoman/tests/.gnupg/trustdb.gpg
new file mode 100644
index 000000000..db5b1023b
--- /dev/null
+++ b/repoman/lib/repoman/tests/.gnupg/trustdb.gpg
Binary files differ
diff --git a/repoman/lib/repoman/tests/runTests.py b/repoman/lib/repoman/tests/runTests.py
index e23ded192..4a081551e 100644
--- a/repoman/lib/repoman/tests/runTests.py
+++ b/repoman/lib/repoman/tests/runTests.py
@@ -10,6 +10,8 @@ import grp
import platform
import pwd
import signal
+import tempfile
+from distutils.dir_util import copy_tree
def debug_signal(signum, frame):
@@ -68,6 +70,14 @@ if insert_bin_path:
path.insert(0, PORTAGE_BIN_PATH)
os.environ["PATH"] = ":".join(path)
+# Copy GPG test keys to temporary directory
+gpg_path = tempfile.mkdtemp(prefix="gpg_")
+
+copy_tree(os.path.join(os.path.dirname(os.path.realpath(__file__)), ".gnupg"), gpg_path)
+
+os.chmod(gpg_path, 0o700)
+os.environ["PORTAGE_GNUPGHOME"] = gpg_path
+
if __name__ == "__main__":
try:
sys.exit(tests.main())
diff --git a/repoman/setup.py b/repoman/setup.py
index 17c9c5e8a..8231c8f75 100755
--- a/repoman/setup.py
+++ b/repoman/setup.py
@@ -13,7 +13,7 @@ try:
from setuptools.command.install_scripts import install_scripts
from setuptools.command.sdist import sdist
from setuptools.dep_util import newer
- from setuptools.dir_util import mkpath, remove_tree
+ from setuptools.dir_util import mkpath, remove_tree, copy_tree
from setuptools.util import change_root, subst_vars
except ImportError:
from distutils.core import setup, Command
@@ -26,7 +26,7 @@ except ImportError:
from distutils.command.install_scripts import install_scripts
from distutils.command.sdist import sdist
from distutils.dep_util import newer
- from distutils.dir_util import mkpath, remove_tree
+ from distutils.dir_util import mkpath, remove_tree, copy_tree
from distutils.util import change_root, subst_vars
import codecs
@@ -425,6 +425,14 @@ class test(Command):
def run(self):
self.run_command("build_tests")
+ # copy GPG test keys
+ copy_tree(
+ os.path.join(
+ self.build_lib, "..", "..", "lib", "repoman", "tests", ".gnupg"
+ ),
+ os.path.join(self.build_lib, "repoman", "tests", ".gnupg"),
+ )
+ os.chmod(os.path.join(self.build_lib, "repoman", "tests", ".gnupg"), 0o700)
subprocess.check_call(
[
sys.executable,
diff --git a/setup.py b/setup.py
index 1157e5824..9e6b4cdb2 100755
--- a/setup.py
+++ b/setup.py
@@ -14,7 +14,7 @@ try:
from setuptools.command.install_scripts import install_scripts
from setuptools.command.sdist import sdist
from setuptools.dep_util import newer
- from setuptools.dir_util import mkpath, remove_tree
+ from setuptools.dir_util import mkpath, remove_tree, copy_tree
from setuptools.util import change_root, subst_vars
except ImportError:
from distutils.core import setup, Command, Extension
@@ -28,7 +28,7 @@ except ImportError:
from distutils.command.install_scripts import install_scripts
from distutils.command.sdist import sdist
from distutils.dep_util import newer
- from distutils.dir_util import mkpath, remove_tree
+ from distutils.dir_util import mkpath, remove_tree, copy_tree
from distutils.util import change_root, subst_vars
import codecs
@@ -700,6 +700,16 @@ class test(Command):
def run(self):
self.run_command("build_tests")
+
+ # copy GPG test keys
+ copy_tree(
+ os.path.join(
+ self.build_lib, "..", "..", "lib", "portage", "tests", ".gnupg"
+ ),
+ os.path.join(self.build_lib, "portage", "tests", ".gnupg"),
+ )
+ os.chmod(os.path.join(self.build_lib, "portage", "tests", ".gnupg"), 0o700)
+
subprocess.check_call(
[
sys.executable,
@@ -805,7 +815,7 @@ def get_data_files(regular_files, venv_files):
setup(
name="portage",
- version="3.0.27",
+ version="3.0.30",
url="https://wiki.gentoo.org/wiki/Project:Portage",
project_urls={
"Release Notes": "https://gitweb.gentoo.org/proj/portage.git/plain/RELEASE-NOTES",
@@ -833,12 +843,16 @@ setup(
["$portage_setsdir", ["cnf/sets/portage.conf"]],
["$docdir", ["NEWS", "RELEASE-NOTES"]],
["$portage_base/bin", ["bin/deprecated-path"]],
- ["$sysconfdir/portage/repo.postsync.d", ["cnf/repo.postsync.d/example"]],
+ ["$portage_confdir/repo.postsync.d", ["cnf/repo.postsync.d/example"]],
],
[
("etc", "cnf", ("etc-update.conf", "dispatch-conf.conf")),
("etc/logrotate.d", "cnf/logrotate.d", ("elog-save-summary",)),
- ("etc/portage/repo.postsync.d", "cnf/repo.postsync.d", ("example",)),
+ (
+ "share/portage/config/repo.postsync.d",
+ "cnf/repo.postsync.d",
+ ("example",),
+ ),
(
"share/portage/config",
"cnf",
diff --git a/src/portage_util_file_copy_reflink_linux.c b/src/portage_util_file_copy_reflink_linux.c
index c6affe57a..b00b57952 100644
--- a/src/portage_util_file_copy_reflink_linux.c
+++ b/src/portage_util_file_copy_reflink_linux.c
@@ -261,13 +261,14 @@ _reflink_linux_file_copy(PyObject *self, PyObject *args)
&offset_out,
len);
- if (copyfunc_ret < 0) {
+ if (copyfunc_ret <= 0) {
error = errno;
- if ((errno == EXDEV || errno == ENOSYS || errno == EOPNOTSUPP) &&
+ if ((errno == EXDEV || errno == ENOSYS || errno == EOPNOTSUPP || copyfunc_ret == 0) &&
copyfunc == cfr_wrapper) {
/* Use sendfile instead of copy_file_range for
* cross-device copies, or when the copy_file_range
- * syscall is not available (less than Linux 4.5).
+ * syscall is not available (less than Linux 4.5),
+ * or when copy_file_range copies zero bytes.
*/
error = 0;
copyfunc = sf_wrapper;