diff options
Diffstat (limited to 'lib/portage/tests/emerge')
| -rw-r--r-- | lib/portage/tests/emerge/conftest.py | 859 | ||||
| -rw-r--r-- | lib/portage/tests/emerge/meson.build | 16 | ||||
| -rw-r--r-- | lib/portage/tests/emerge/test_actions.py | 68 | ||||
| -rw-r--r-- | lib/portage/tests/emerge/test_baseline.py | 221 | ||||
| -rw-r--r-- | lib/portage/tests/emerge/test_binpkg_fetch.py | 226 | ||||
| -rw-r--r-- | lib/portage/tests/emerge/test_config_protect.py | 33 | ||||
| -rw-r--r-- | lib/portage/tests/emerge/test_emerge_blocker_file_collision.py | 19 | ||||
| -rw-r--r-- | lib/portage/tests/emerge/test_emerge_slot_abi.py | 26 | ||||
| -rw-r--r-- | lib/portage/tests/emerge/test_libc_dep_inject.py | 552 | ||||
| -rw-r--r-- | lib/portage/tests/emerge/test_simple.py | 704 |
10 files changed, 1991 insertions, 733 deletions
diff --git a/lib/portage/tests/emerge/conftest.py b/lib/portage/tests/emerge/conftest.py new file mode 100644 index 000000000..ce86dc4fc --- /dev/null +++ b/lib/portage/tests/emerge/conftest.py @@ -0,0 +1,859 @@ +# Copyright 2023 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +import argparse +import shlex +from typing import Optional, Callable # , Self + +from portage.const import ( + SUPPORTED_GENTOO_BINPKG_FORMATS, + BASH_BINARY, + BINREPOS_CONF_FILE, +) +from portage.tests.resolver.ResolverPlayground import ResolverPlayground +from portage.cache.mappings import Mapping +from portage.tests.util.test_socks5 import AsyncHTTPServer +from portage import os +from portage import shutil +from portage.util.futures import asyncio +from portage.tests import cnf_bindir, cnf_sbindir +from portage.process import find_binary +from portage.util import find_updated_config_files +import portage + +import pytest + + +_INSTALL_SOMETHING = """ +S="${WORKDIR}" + +pkg_pretend() { + einfo "called pkg_pretend for $CATEGORY/$PF" +} + +src_install() { + einfo "installing something..." + insinto /usr/lib/${P} + echo "blah blah blah" > "${T}"/regular-file + doins "${T}"/regular-file + dosym regular-file /usr/lib/${P}/symlink || die + + # Test CONFIG_PROTECT + insinto /etc + newins "${T}"/regular-file ${PN}-${SLOT%/*} + + # Test code for bug #381629, using a copyright symbol encoded with latin-1. + # We use $(printf "\\xa9") rather than $'\\xa9', since printf apparently + # works in any case, while $'\\xa9' transforms to \\xef\\xbf\\xbd under + # some conditions. TODO: Find out why it transforms to \\xef\\xbf\\xbd when + # running tests for Python 3.2 (even though it's bash that is ultimately + # responsible for performing the transformation). + local latin_1_dir=/usr/lib/${P}/latin-1-$(printf "\\xa9")-directory + insinto "${latin_1_dir}" + echo "blah blah blah" > "${T}"/latin-1-$(printf "\\xa9")-regular-file || die + doins "${T}"/latin-1-$(printf "\\xa9")-regular-file + dosym latin-1-$(printf "\\xa9")-regular-file ${latin_1_dir}/latin-1-$(printf "\\xa9")-symlink || die + + call_has_and_best_version +} + +pkg_config() { + einfo "called pkg_config for $CATEGORY/$PF" +} + +pkg_info() { + einfo "called pkg_info for $CATEGORY/$PF" +} + +pkg_preinst() { + if ! ___eapi_best_version_and_has_version_support_-b_-d_-r; then + # The BROOT variable is unset during pkg_* phases for EAPI 7, + # therefore best/has_version -b is expected to fail if we attempt + # to call it for EAPI 7 here. + call_has_and_best_version + fi +} + +call_has_and_best_version() { + local root_arg + if ___eapi_best_version_and_has_version_support_-b_-d_-r; then + root_arg="-b" + else + root_arg="--host-root" + fi + einfo "called ${EBUILD_PHASE_FUNC} for $CATEGORY/$PF" + einfo "EPREFIX=${EPREFIX}" + einfo "PORTAGE_OVERRIDE_EPREFIX=${PORTAGE_OVERRIDE_EPREFIX}" + einfo "ROOT=${ROOT}" + einfo "EROOT=${EROOT}" + einfo "SYSROOT=${SYSROOT}" + einfo "ESYSROOT=${ESYSROOT}" + einfo "BROOT=${BROOT}" + # Test that has_version and best_version work correctly with + # prefix (involves internal ROOT -> EROOT calculation in order + # to support ROOT override via the environment with EAPIs 3 + # and later which support prefix). + if has_version $CATEGORY/$PN:$SLOT ; then + einfo "has_version detects an installed instance of $CATEGORY/$PN:$SLOT" + einfo "best_version reports that the installed instance is $(best_version $CATEGORY/$PN:$SLOT)" + else + einfo "has_version does not detect an installed instance of $CATEGORY/$PN:$SLOT" + fi + if [[ ${EPREFIX} != ${PORTAGE_OVERRIDE_EPREFIX} ]] ; then + if has_version ${root_arg} $CATEGORY/$PN:$SLOT ; then + einfo "has_version ${root_arg} detects an installed instance of $CATEGORY/$PN:$SLOT" + einfo "best_version ${root_arg} reports that the installed instance is $(best_version ${root_arg} $CATEGORY/$PN:$SLOT)" + else + einfo "has_version ${root_arg} does not detect an installed instance of $CATEGORY/$PN:$SLOT" + fi + fi +} + +""" + +_AVAILABLE_EBUILDS = { + "dev-libs/A-1": { + "EAPI": "5", + "IUSE": "+flag", + "KEYWORDS": "x86", + "LICENSE": "GPL-2", + "MISC_CONTENT": _INSTALL_SOMETHING, + "RDEPEND": "flag? ( dev-libs/B[flag] )", + }, + "dev-libs/B-1": { + "EAPI": "5", + "IUSE": "+flag", + "KEYWORDS": "x86", + "LICENSE": "GPL-2", + "MISC_CONTENT": _INSTALL_SOMETHING, + }, + "dev-libs/C-1": { + "EAPI": "7", + "KEYWORDS": "~x86", + "RDEPEND": "dev-libs/D[flag]", + "MISC_CONTENT": _INSTALL_SOMETHING, + }, + "dev-libs/D-1": { + "EAPI": "7", + "KEYWORDS": "~x86", + "IUSE": "flag", + "MISC_CONTENT": _INSTALL_SOMETHING, + }, + "virtual/foo-0": { + "EAPI": "5", + "KEYWORDS": "x86", + "LICENSE": "GPL-2", + }, +} + +_INSTALLED_EBUILDS = { + "dev-libs/A-1": { + "EAPI": "5", + "IUSE": "+flag", + "KEYWORDS": "x86", + "LICENSE": "GPL-2", + "RDEPEND": "flag? ( dev-libs/B[flag] )", + "USE": "flag", + }, + "dev-libs/B-1": { + "EAPI": "5", + "IUSE": "+flag", + "KEYWORDS": "x86", + "LICENSE": "GPL-2", + "USE": "flag", + }, + "dev-libs/depclean-me-1": { + "EAPI": "5", + "IUSE": "", + "KEYWORDS": "x86", + "LICENSE": "GPL-2", + "USE": "", + }, + "app-misc/depclean-me-1": { + "EAPI": "5", + "IUSE": "", + "KEYWORDS": "x86", + "LICENSE": "GPL-2", + "RDEPEND": "dev-libs/depclean-me", + "USE": "", + }, +} + + +_BASELINE_COMMAND_SEQUENCE = [ + "emerge -1 dev-libs/A -v dev-libs/B", + "emerge with quickpkg direct", + "env-update", + "portageq envvar", + "etc-update", + "dispatch-conf", + "emerge --version", + "emerge --info", + "emerge --info --verbose", + "emerge --list-sets", + "emerge --check-news", + "emerge --regen/--metadata", + "misc package operations", + "binhost emerge", +] + +PORTAGE_PYTHON = portage._python_interpreter +NOOP = lambda: ... + + +class PortageCommand: + """A class that represents a baseline test case command, + including handling of environment and one-use arguments. + """ + + command = None + name = None + + def __init__( + self, + *args: tuple[str], + env_mod: Optional[dict[str, str]] = None, + preparation: Optional[Callable[[], None]] = None, + post_command: Optional[Callable[[], None]] = None, + ) -> None: + self.args = args + self.env_mod = env_mod + self.preparation = preparation + self.post_command = post_command + + def __iter__(self): + """To be able to call a function with ``*command`` as argument.""" + yield self + + @property + def env(self) -> dict[str, str]: + """This property returns the environment intended to be used + with the current test command, including possible modifications. + """ + try: + base_environment = self.base_environment + except AttributeError: + base_environment = {} + else: + base_environment = base_environment.copy() + if self.env_mod: + base_environment.update(self.env_mod) + return base_environment + + def __call__(self): # -> Self: + if self.preparation: + self.preparation() + try: + tuple_command = self.command + self.args + except TypeError: + # In case self.command is a string: + tuple_command = (self.command,) + self.args + return tuple_command + + def __bool__(self) -> bool: + return bool(self.command) + + def check_command_result(self) -> None: + if self.post_command: + self.post_command() + + +class PortageCommandSequence: + def __init__(self, *commands): + self.commands = commands + + def __iter__(self): + yield from self.commands + + +class Emerge(PortageCommand): + name = "emerge" + command = (PORTAGE_PYTHON, "-b", "-Wd", os.path.join(str(cnf_bindir), name)) + + +class Noop(PortageCommand): + name = "No-op" + + +class EnvUpdate(PortageCommand): + name = "env-update" + command = (PORTAGE_PYTHON, "-b", "-Wd", os.path.join(str(cnf_sbindir), name)) + + +class DispatchConf(PortageCommand): + name = "dispatch-conf" + command = ( + PORTAGE_PYTHON, + "-b", + "-Wd", + os.path.join(str(cnf_sbindir), name), + ) + + +class Ebuild(PortageCommand): + name = "ebuild" + command = (PORTAGE_PYTHON, "-b", "-Wd", os.path.join(str(cnf_bindir), name)) + + +class Egencache(PortageCommand): + name = "egencache" + command = ( + PORTAGE_PYTHON, + "-b", + "-Wd", + os.path.join(str(cnf_bindir), name), + ) + + +class Emaint(PortageCommand): + name = "emaint" + command = (PORTAGE_PYTHON, "-b", "-Wd", os.path.join(str(cnf_sbindir), name)) + + +class EtcUpdate(PortageCommand): + name = "etc-update" + command = (BASH_BINARY, os.path.join(str(cnf_sbindir), name)) + + +class Fixpackages(PortageCommand): + name = "fixpackages" + command = ( + PORTAGE_PYTHON, + "-b", + "-Wd", + os.path.join(str(cnf_sbindir), name), + ) + + +class Portageq(PortageCommand): + name = "portageq" + command = ( + PORTAGE_PYTHON, + "-b", + "-Wd", + os.path.join(str(cnf_bindir), name), + ) + + +class Quickpkg(PortageCommand): + name = "quickpkg" + command = ( + PORTAGE_PYTHON, + "-b", + "-Wd", + os.path.join(str(cnf_bindir), name), + ) + + +class Regenworld(PortageCommand): + name = "regenworld" + command = ( + PORTAGE_PYTHON, + "-b", + "-Wd", + os.path.join(str(cnf_sbindir), name), + ) + + +def pytest_generate_tests(metafunc): + if "baseline_command" in metafunc.fixturenames: + metafunc.parametrize( + "baseline_command", _BASELINE_COMMAND_SEQUENCE, indirect=True + ) + + +def _have_python_xml(): + try: + __import__("xml.etree.ElementTree") + __import__("xml.parsers.expat").parsers.expat.ExpatError + except (AttributeError, ImportError): + return False + return True + + +def _check_foo_file(pkgdir, filename, must_exist) -> None: + assert ( + os.path.exists(os.path.join(pkgdir, "virtual", "foo", filename)) == must_exist + ) + + +def _check_number_of_protected_files(must_have, eroot, config_protect) -> None: + assert must_have == len( + list(find_updated_config_files(eroot, shlex.split(config_protect))) + ) + + +class BinhostContentMap(Mapping): + def __init__(self, remote_path, local_path): + self._remote_path = remote_path + self._local_path = local_path + + def __getitem__(self, request_path): + safe_path = os.path.normpath(request_path) + if not safe_path.startswith(self._remote_path + "/"): + raise KeyError(request_path) + local_path = os.path.join( + self._local_path, safe_path[len(self._remote_path) + 1 :] + ) + try: + with open(local_path, "rb") as f: + return f.read() + except OSError: + raise KeyError(request_path) + + +@pytest.fixture(scope="module") +def async_loop(): + yield asyncio._wrap_loop() + + +@pytest.fixture(params=SUPPORTED_GENTOO_BINPKG_FORMATS, scope="function") +def playground(request, tmp_path_factory): + """Fixture that provides instances of ``ResolverPlayground`` + each one with one supported value for ``BINPKG_FORMAT``.""" + binpkg_format = request.param + playground = ResolverPlayground( + ebuilds=_AVAILABLE_EBUILDS, + installed=_INSTALLED_EBUILDS, + debug=False, + user_config={ + "make.conf": (f'BINPKG_FORMAT="{binpkg_format}"',), + }, + eprefix=str(tmp_path_factory.mktemp("eprefix", numbered=True)), + ) + yield playground + playground.cleanup() + + +@pytest.fixture() +def binhost(playground, async_loop): + settings = playground.settings + eprefix = settings["EPREFIX"] + binhost_dir = os.path.join(eprefix, "binhost") + binhost_address = "127.0.0.1" + binhost_remote_path = "/binhost" + binhost_server = AsyncHTTPServer( + binhost_address, BinhostContentMap(binhost_remote_path, binhost_dir), async_loop + ).__enter__() + binhost_uri = "http://{address}:{port}{path}".format( + address=binhost_address, + port=binhost_server.server_port, + path=binhost_remote_path, + ) + yield {"server": binhost_server, "uri": binhost_uri, "dir": binhost_dir} + binhost_server.__exit__(None, None, None) + + +@pytest.fixture() +def _generate_all_baseline_commands(playground, binhost): + """This fixture generates all the commands that + ``test_portage_baseline`` will use. + + But, don't use this fixture directly, instead, use the + ``baseline_command`` fixture. That improves performance a bit due to + pytest caching (?). + + .. note:: + + To add a new command, define it in the local ``test_commands`` + dict, if not yet defined, and add its key at the correct position + in the ``_BASELINE_COMMAND_SEQUENCE`` list. + """ + settings = playground.settings + eprefix = settings["EPREFIX"] + eroot = settings["EROOT"] + trees = playground.trees + pkgdir = playground.pkgdir + portdb = trees[eroot]["porttree"].dbapi + test_repo_location = settings.repositories["test_repo"].location + var_cache_edb = os.path.join(eprefix, "var", "cache", "edb") + cachedir = os.path.join(var_cache_edb, "dep") + cachedir_pregen = os.path.join(test_repo_location, "metadata", "md5-cache") + + rm_binary = find_binary("rm") + assert rm_binary is not None, "rm command not found" + rm_cmd = (rm_binary,) + + egencache_extra_args = [] + if _have_python_xml(): + egencache_extra_args.append("--update-use-local-desc") + + test_ebuild = portdb.findname("dev-libs/A-1") + assert test_ebuild is not None + + cross_prefix = os.path.join(eprefix, "cross_prefix") + cross_root = os.path.join(eprefix, "cross_root") + cross_eroot = os.path.join(cross_root, eprefix.lstrip(os.sep)) + + binpkg_format = settings.get("BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0]) + assert binpkg_format in ("xpak", "gpkg") + if binpkg_format == "xpak": + foo_filename = "foo-0-1.xpak" + elif binpkg_format == "gpkg": + foo_filename = "foo-0-1.gpkg.tar" + + test_commands = {} + + if hasattr(argparse.ArgumentParser, "parse_intermixed_args"): + parse_intermixed_command = Emerge( + "--oneshot", + "dev-libs/A", + "-v", + "dev-libs/A", + ) + else: + parse_intermixed_command = Noop() + test_commands["emerge -1 dev-libs/A -v dev-libs/B"] = parse_intermixed_command + + quickpkg_direct_seq = [ + Emerge( + "--usepkgonly", + "--root", + cross_root, + "--quickpkg-direct=y", + "--quickpkg-direct-root", + "/", + "dev-libs/A", + ), + # v needs ^ + Emerge( + "--usepkgonly", + "--quickpkg-direct=y", + "--quickpkg-direct-root", + cross_root, + "dev-libs/A", + ), + ] + test_commands["emerge with quickpkg direct"] = PortageCommandSequence( + *quickpkg_direct_seq + ) + + test_commands["env-update"] = EnvUpdate() + test_commands["portageq envvar"] = Portageq( + "envvar", + "-v", + "CONFIG_PROTECT", + "EROOT", + "PORTAGE_CONFIGROOT", + "PORTAGE_TMPDIR", + "USERLAND", + ) + test_commands["etc-update"] = EtcUpdate() + test_commands["dispatch-conf"] = DispatchConf() + test_commands["emerge --version"] = Emerge("--version") + test_commands["emerge --info"] = Emerge("--info") + test_commands["emerge --info --verbose"] = Emerge("--info", "--verbose") + test_commands["emerge --list-sets"] = Emerge("--list-sets") + test_commands["emerge --check-news"] = Emerge("--check-news") + + def _rm_cachedir(): + shutil.rmtree(cachedir) + + def _rm_cachedir_and_pregen(): + _rm_cachedir() + shutil.rmtree(cachedir_pregen) + + regen_seq = [ + Emerge("--regen", preparation=_rm_cachedir_and_pregen), + Emerge( + "--regen", + env_mod={"FEATURES": "metadata-transfer"}, + preparation=_rm_cachedir, + ), + Egencache( + "--repo", + "test_repo", + "--repositories-configuration", + playground.settings.repositories.config_string(), + "--update", + *egencache_extra_args, + preparation=_rm_cachedir, + ), + Emerge("--metadata", env_mod={"FEATURES": "metadata-transfer"}), + Emerge( + "--metadata", + env_mod={"FEATURES": "metadata-transfer"}, + preparation=_rm_cachedir, + ), + Emerge("--metadata"), + Emerge("--oneshot", "virtual/foo", preparation=_rm_cachedir), + Emerge( + "--unmerge", + "virtual/foo", + env_mod={"FEATURES": "unmerge-backup"}, + preparation=lambda: _check_foo_file(pkgdir, foo_filename, must_exist=False), + ), + Emerge( + "--pretend", + "dev-libs/A", + preparation=lambda: _check_foo_file(pkgdir, foo_filename, must_exist=True), + ), + ] + test_commands["emerge --regen/--metadata"] = PortageCommandSequence(*regen_seq) + + abcd_seq = [ + Ebuild( + test_ebuild, + "manifest", + "clean", + "package", + "merge", + ), + Emerge( + "--pretend", + "--tree", + "--complete-graph", + "dev-libs/A", + ), + Emerge("-p", "dev-libs/B"), + Emerge( + "-p", + "--newrepo", + "dev-libs/B", + ), + Emerge("-B", "dev-libs/B"), + Emerge( + "--oneshot", + "--usepkg", + "dev-libs/B", + ), + # trigger clean prior to pkg_pretend as in bug #390711 + Ebuild(test_ebuild, "unpack"), + Emerge("--oneshot", "dev-libs/A"), + Emerge("--noreplace", "dev-libs/A"), + Emerge( + "--config", + "dev-libs/A", + ), + Emerge( + "--info", + "dev-libs/A", + "dev-libs/B", + ), + Emerge( + "--pretend", + "--depclean", + "--verbose", + "dev-libs/B", + ), + Emerge("--pretend", "--depclean"), + Emerge( + "--depclean", + ), + # Test bug #523684, where a file renamed or removed by the + # admin forces replacement files to be merged with config + # protection. + Quickpkg( + "--include-config", + "y", + "dev-libs/A", + post_command=lambda: _check_number_of_protected_files( + 0, eroot, settings["CONFIG_PROTECT"] + ), + ), + Emerge("--noreplace", "dev-libs/A"), + Emerge( + "--usepkgonly", + "dev-libs/A", + preparation=lambda: os.unlink(os.path.join(eprefix, "etc", "A-0")), + post_command=lambda: _check_number_of_protected_files( + 1, eroot, settings["CONFIG_PROTECT"] + ), + ), + Emaint("--check", "all"), + Emaint("--fix", "all"), + Fixpackages(), + Regenworld(), + Portageq( + "match", + eroot, + "dev-libs/A", + ), + Portageq( + "best_visible", + eroot, + "dev-libs/A", + ), + Portageq( + "best_visible", + eroot, + "binary", + "dev-libs/A", + ), + Portageq( + "contents", + eroot, + "dev-libs/A-1", + ), + Portageq( + "metadata", + eroot, + "ebuild", + "dev-libs/A-1", + "EAPI", + "IUSE", + "RDEPEND", + ), + Portageq( + "metadata", + eroot, + "binary", + "dev-libs/A-1", + "EAPI", + "USE", + "RDEPEND", + ), + Portageq( + "metadata", + eroot, + "installed", + "dev-libs/A-1", + "EAPI", + "USE", + "RDEPEND", + ), + Portageq( + "owners", + eroot, + eroot + "usr", + ), + Emerge("-p", eroot + "usr"), + Emerge( + "-p", + "--unmerge", + "-q", + eroot + "usr", + ), + Emerge( + "--unmerge", + "--quiet", + "dev-libs/A", + ), + Emerge( + "-C", + "--quiet", + "dev-libs/B", + ), + # autounmask: + # If EMERGE_DEFAULT_OPTS contains --autounmask=n, then --autounmask + # must be specified with --autounmask-continue. + Emerge( + "--autounmask", + "--autounmask-continue", + "dev-libs/C", + env_mod={"EMERGE_DEFAULT_OPTS": "--autounmask=n"}, + ), + # Verify that the above --autounmask-continue command caused + # USE=flag to be applied correctly to dev-libs/D. + Portageq( + "match", + eroot, + "dev-libs/D[flag]", + ), + ] + test_commands["misc package operations"] = PortageCommandSequence(*abcd_seq) + + cross_prefix_seq = [ + # Test cross-prefix usage, including chpathtool for binpkgs. + # EAPI 7 + Emerge("dev-libs/C", env_mod={"EPREFIX": cross_prefix}), + Portageq( + "has_version", cross_prefix, "dev-libs/C", env_mod={"EPREFIX": cross_prefix} + ), + Portageq( + "has_version", cross_prefix, "dev-libs/D", env_mod={"EPREFIX": cross_prefix} + ), + Emerge("dev-libs/D", env_mod={"ROOT": cross_root}), + Portageq( + "has_version", + cross_eroot, + "dev-libs/D", + ), + # EAPI 5 + Emerge("--usepkgonly", "dev-libs/A", env_mod={"EPREFIX": cross_prefix}), + Portageq( + "has_version", cross_prefix, "dev-libs/A", env_mod={"EPREFIX": cross_prefix} + ), + Portageq( + "has_version", cross_prefix, "dev-libs/B", env_mod={"EPREFIX": cross_prefix} + ), + Emerge("-C", "--quiet", "dev-libs/B", env_mod={"EPREFIX": cross_prefix}), + Emerge("-C", "--quiet", "dev-libs/A", env_mod={"EPREFIX": cross_prefix}), + Emerge("dev-libs/A", env_mod={"EPREFIX": cross_prefix}), + # Test ROOT support + Emerge("dev-libs/B", env_mod={"ROOT": cross_root}), + Portageq( + "has_version", + cross_eroot, + "dev-libs/B", + ), + ] + test_commands["misc operations with eprefix"] = PortageCommandSequence( + *cross_prefix_seq + ) + + # Test binhost support if FETCHCOMMAND is available. + binrepos_conf_file = os.path.join(os.sep, eprefix, BINREPOS_CONF_FILE) + binhost_uri = binhost["uri"] + binhost_dir = binhost["dir"] + with open(binrepos_conf_file, "w") as f: + f.write("[test-binhost]\n") + f.write(f"sync-uri = {binhost_uri}\n") + fetchcommand = shlex.split(settings["FETCHCOMMAND"]) + fetch_bin = portage.process.find_binary(fetchcommand[0]) + + if fetch_bin is None: + test_commands["binhost emerge"] = Noop() + else: + # The next emerge has been added to split this test from the rest: + make_package = Emerge("-e", "--buildpkg", "dev-libs/A") + getbinpkgonly = Emerge( + "-e", + "--getbinpkgonly", + "dev-libs/A", + preparation=lambda: os.rename(pkgdir, binhost_dir), + ) + + # Remove binrepos.conf and test PORTAGE_BINHOST. + def _rm_pkgdir_and_rm_binrepos_conf_file(): + shutil.rmtree(pkgdir) + os.unlink(binrepos_conf_file) + + getbinpkgonly_fetchonly = Emerge( + "-fe", + "--getbinpkgonly", + "dev-libs/A", + env_mod={"PORTAGE_BINHOST": binhost_uri}, + preparation=_rm_pkgdir_and_rm_binrepos_conf_file, + ) + + # Test bug 920537 binrepos.conf with local file src-uri. + def _rm_pkgdir_and_create_binrepos_conf_with_file_uri(): + shutil.rmtree(pkgdir) + with open(binrepos_conf_file, "w") as f: + f.write("[test-binhost]\n") + f.write(f"sync-uri = file://{binhost_dir}\n") + + getbinpkgonly_file_uri = Emerge( + "-fe", + "--getbinpkgonly", + "dev-libs/A", + preparation=_rm_pkgdir_and_create_binrepos_conf_with_file_uri, + ) + + fetch_sequence = PortageCommandSequence( + make_package, getbinpkgonly, getbinpkgonly_fetchonly, getbinpkgonly_file_uri + ) + test_commands["binhost emerge"] = fetch_sequence + yield test_commands + + +@pytest.fixture() +def baseline_command(request, _generate_all_baseline_commands): + """A fixture that provides the commands to perform a baseline + functional test of portage. It uses another fixture, namely + ``_generate_all_baseline_commands``. + Pytest caches the fixtures and there is a little performance + improvement if the commands are generated only once.. + """ + return _generate_all_baseline_commands[request.param] diff --git a/lib/portage/tests/emerge/meson.build b/lib/portage/tests/emerge/meson.build new file mode 100644 index 000000000..0e0a41974 --- /dev/null +++ b/lib/portage/tests/emerge/meson.build @@ -0,0 +1,16 @@ +py.install_sources( + [ + 'test_actions.py', + 'test_binpkg_fetch.py', + 'test_config_protect.py', + 'test_emerge_blocker_file_collision.py', + 'test_emerge_slot_abi.py', + 'test_global_updates.py', + 'test_baseline.py', + 'test_libc_dep_inject.py', + '__init__.py', + '__test__.py', + ], + subdir : 'portage/tests/emerge', + pure : not native_extensions +) diff --git a/lib/portage/tests/emerge/test_actions.py b/lib/portage/tests/emerge/test_actions.py new file mode 100644 index 000000000..cdc087a8e --- /dev/null +++ b/lib/portage/tests/emerge/test_actions.py @@ -0,0 +1,68 @@ +# Copyright 2022 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +from unittest.mock import MagicMock, patch + +from _emerge.actions import get_libc_version, run_action + +from portage.const import LIBC_PACKAGE_ATOM +from portage.dbapi.virtual import fakedbapi +from portage.dep import Atom +from portage.tests import TestCase + + +class RunActionTestCase(TestCase): + """This class' purpose is to encompass UTs for ``actions.run_action``. + Since that function is extremely long (at least on Sep. 2022; + hopefully the situation gets better with the time), the tests in this + ``TestCase`` contain plenty of mocks/patches. + Hopefully, with time and effort, the ``run_action`` function (and others + in the module) are refactored to make testing easier and more robust. + + A side effect of the mocking approach is a strong dependency on the + details of the implementation. That can be improved if functions + are smaller and do a well defined small set of tasks. Another call to + refactoring... + If the implementation changes, the mocks can be adjusted to play its + role. + """ + + @patch("_emerge.actions.profile_check") + @patch("_emerge.actions.adjust_configs") + @patch("_emerge.actions.apply_priorities") + def test_binary_trees_populate_called(self, papply, padjust, profile_ckeck): + """Ensure that ``binarytree.populate`` API is correctly used. + The point of this test is to ensure that the ``populate`` method + is called as expected: since it is the first time that ``populate`` + is called, it must use ``getbinpkg_refresh=True``. + """ + config = MagicMock() + config.action = None + config.opts = {"--quiet": True, "--usepkg": True, "--package-moves": "n"} + bt = MagicMock() + tree = {"bintree": bt} + trees = {"first": tree} + config.trees = trees + + run_action(config) + + bt.populate.assert_called_once_with( + getbinpkgs=False, getbinpkg_refresh=True, pretend=False + ) + + def testGetSystemLibc(self): + """ + Check that get_libc_version extracts the right version string + from the provider LIBC_PACKAGE_ATOM for emerge --info and friends. + """ + settings = MagicMock() + + settings.getvirtuals.return_value = { + LIBC_PACKAGE_ATOM: [Atom("=sys-libs/musl-1.2.3")] + } + settings.__getitem__.return_value = {} + + vardb = fakedbapi(settings) + vardb.cpv_inject("sys-libs/musl-1.2.3", {"SLOT": "0"}) + + self.assertEqual(get_libc_version(vardb), ["musl-1.2.3"]) diff --git a/lib/portage/tests/emerge/test_baseline.py b/lib/portage/tests/emerge/test_baseline.py new file mode 100644 index 000000000..eb4a3372d --- /dev/null +++ b/lib/portage/tests/emerge/test_baseline.py @@ -0,0 +1,221 @@ +# Copyright 2011-2021, 2023 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +"""This module defines a baseline for portage's functionality. + +Multiple portage commands are executed in a sequence in a playground +(see the ``baseline_command`` fixture in ``conftest.py``). + +All the commands are triggered from the ``test_portage_baseline`` test. +That test is marked with:: + + @pytest.mark.ft + +so that it can selected with that marker, i.e.:: + + pytest -m ft + +``ft`` stands for *functional test*, since that's what it is, a +functional or end-to-end test, ensuring some functionality of portage. + +The test also works with pytest-xdist, e.g.:: + + pytest -m ft -n 8 + +""" + +import subprocess + +import pytest + +import portage +from portage import os +from portage.const import ( + PORTAGE_PYM_PATH, + USER_CONFIG_PATH, +) +from portage.process import find_binary +from portage.tests import cnf_etc_path +from portage.util import ensure_dirs +from portage.util.futures import asyncio + + +_METADATA_XML_FILES = ( + ( + "dev-libs/A", + { + "flags": "<flag name='flag'>Description of how USE='flag' affects this package</flag>", + }, + ), + ( + "dev-libs/B", + { + "flags": "<flag name='flag'>Description of how USE='flag' affects this package</flag>", + }, + ), +) + +_1Q_2010_UPDATE = """ +slotmove =app-doc/pms-3 2 3 +move dev-util/git dev-vcs/git +""" + + +@pytest.mark.ft +def test_portage_baseline(async_loop, playground, binhost, baseline_command): + async_loop.run_until_complete( + asyncio.ensure_future( + _async_test_baseline( + playground, + binhost, + baseline_command, + ), + loop=async_loop, + ) + ) + + +async def _async_test_baseline(playground, binhost, commands): + debug = playground.debug + settings = playground.settings + trees = playground.trees + eprefix = settings["EPREFIX"] + + test_repo_location = settings.repositories["test_repo"].location + var_cache_edb = os.path.join(eprefix, "var", "cache", "edb") + cachedir = os.path.join(var_cache_edb, "dep") + cachedir_pregen = os.path.join(test_repo_location, "metadata", "md5-cache") + + cross_prefix = os.path.join(eprefix, "cross_prefix") + cross_root = os.path.join(eprefix, "cross_root") + cross_eroot = os.path.join(cross_root, eprefix.lstrip(os.sep)) + + distdir = playground.distdir + pkgdir = playground.pkgdir + fake_bin = os.path.join(eprefix, "bin") + portage_tmpdir = os.path.join(eprefix, "var", "tmp", "portage") + profile_path = settings.profile_path + user_config_dir = os.path.join(os.sep, eprefix, USER_CONFIG_PATH) + + path = settings.get("PATH") + if path is not None and not path.strip(): + path = None + if path is None: + path = "" + else: + path = ":" + path + path = fake_bin + path + + pythonpath = os.environ.get("PYTHONPATH") + if pythonpath is not None and not pythonpath.strip(): + pythonpath = None + if pythonpath is not None and pythonpath.split(":")[0] == PORTAGE_PYM_PATH: + pass + else: + if pythonpath is None: + pythonpath = "" + else: + pythonpath = ":" + pythonpath + pythonpath = PORTAGE_PYM_PATH + pythonpath + + env = { + "PORTAGE_OVERRIDE_EPREFIX": eprefix, + "CLEAN_DELAY": "0", + "DISTDIR": distdir, + "EMERGE_WARNING_DELAY": "0", + "INFODIR": "", + "INFOPATH": "", + "PATH": path, + "PKGDIR": pkgdir, + "PORTAGE_INST_GID": str(os.getgid()), # str(portage.data.portage_gid), + "PORTAGE_INST_UID": str(os.getuid()), # str(portage.data.portage_uid), + "PORTAGE_PYTHON": portage._python_interpreter, + "PORTAGE_REPOSITORIES": settings.repositories.config_string(), + "PORTAGE_TMPDIR": portage_tmpdir, + "PORTAGE_LOGDIR": portage_tmpdir, + "PYTHONDONTWRITEBYTECODE": os.environ.get("PYTHONDONTWRITEBYTECODE", ""), + "PYTHONPATH": pythonpath, + "__PORTAGE_TEST_PATH_OVERRIDE": fake_bin, + } + + if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: + env["__PORTAGE_TEST_HARDLINK_LOCKS"] = os.environ[ + "__PORTAGE_TEST_HARDLINK_LOCKS" + ] + + updates_dir = os.path.join(test_repo_location, "profiles", "updates") + dirs = [ + cachedir, + cachedir_pregen, + cross_eroot, + cross_prefix, + distdir, + fake_bin, + portage_tmpdir, + updates_dir, + user_config_dir, + var_cache_edb, + ] + etc_symlinks = ("dispatch-conf.conf", "etc-update.conf") + # Override things that may be unavailable, or may have portability + # issues when running tests in exotic environments. + # prepstrip - bug #447810 (bash read builtin EINTR problem) + true_symlinks = ["find", "prepstrip", "sed", "scanelf"] + true_binary = find_binary("true") + assert true_binary is not None, "true command not found" + + for d in dirs: + ensure_dirs(d) + for x in true_symlinks: + try: + os.symlink(true_binary, os.path.join(fake_bin, x)) + except FileExistsError: + pass + for x in etc_symlinks: + try: + os.symlink( + os.path.join(str(cnf_etc_path), x), os.path.join(eprefix, "etc", x) + ) + except FileExistsError: + pass + with open(os.path.join(var_cache_edb, "counter"), "wb") as f: + f.write(b"100") + # non-empty system set keeps --depclean quiet + with open(os.path.join(profile_path, "packages"), "w") as f: + f.write("*dev-libs/token-system-pkg") + for cp, xml_data in _METADATA_XML_FILES: + with open(os.path.join(test_repo_location, cp, "metadata.xml"), "w") as f: + f.write(playground.metadata_xml_template % xml_data) + with open(os.path.join(updates_dir, "1Q-2010"), "w") as f: + f.write(_1Q_2010_UPDATE) + if debug: + # The subprocess inherits both stdout and stderr, for + # debugging purposes. + stdout = None + else: + # The subprocess inherits stderr so that any warnings + # triggered by python -Wd will be visible. + stdout = subprocess.PIPE + + for command in commands: + if command: + command.base_environment = env + + proc = await asyncio.create_subprocess_exec( + *command(), env=command.env, stderr=None, stdout=stdout + ) + + if debug: + await proc.wait() + else: + output, _err = await proc.communicate() + await proc.wait() + if proc.returncode != os.EX_OK: + portage.writemsg(output) + + real_command = command.name + args = command.args + assert ( + os.EX_OK == proc.returncode + ), f"'{real_command}' failed with args '{args}'" + command.check_command_result() diff --git a/lib/portage/tests/emerge/test_binpkg_fetch.py b/lib/portage/tests/emerge/test_binpkg_fetch.py new file mode 100644 index 000000000..731711bad --- /dev/null +++ b/lib/portage/tests/emerge/test_binpkg_fetch.py @@ -0,0 +1,226 @@ +# Copyright 2024 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +import shutil +import subprocess +import sys +import tempfile + +import portage +from portage import _unicode_decode, os +from portage.const import ( + PORTAGE_PYM_PATH, + USER_CONFIG_PATH, +) +from portage.process import find_binary +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import ResolverPlayground +from portage.util import ensure_dirs + + +class BinpkgFetchtestCase(TestCase): + def testLocalFilePkgSyncUpdate(self): + """ + Check handling of local file:// sync-uri and unnecessary BUILD_ID + increments (bug #921208). + """ + debug = False + + ebuilds = { + "dev-libs/A-1::local": { + "EAPI": "7", + "SLOT": "0", + }, + } + + playground = ResolverPlayground(ebuilds=ebuilds, debug=debug) + settings = playground.settings + eprefix = settings["EPREFIX"] + eroot = settings["EROOT"] + trees = playground.trees + bindb = trees[eroot]["bintree"].dbapi + var_cache_edb = os.path.join(eprefix, "var", "cache", "edb") + user_config_dir = os.path.join(eprefix, USER_CONFIG_PATH) + + portage_python = portage._python_interpreter + emerge_cmd = ( + portage_python, + "-b", + "-Wd", + os.path.join(str(self.bindir), "emerge"), + ) + + tmppkgdir = tempfile.TemporaryDirectory() + tmppkgdir_suffix = os.path.join(tmppkgdir.name, "binpkg") + + test_commands = ( + # Create a trivial binpkg first. + emerge_cmd + + ( + "--oneshot", + "--verbose", + "--buildpkg", + "dev-libs/A", + ), + # Copy to a new PKGDIR which we'll use as PORTAGE_BINHOST then delete the old PKGDIR. + ( + ( + lambda: shutil.copytree(bindb.bintree.pkgdir, tmppkgdir_suffix) + or True, + ) + ), + ( + ( + lambda: os.unlink( + os.path.join( + bindb.bintree.pkgdir, "dev-libs", "A", "A-1-1.gpkg.tar" + ) + ) + or True, + ) + ), + ) + test_commands_nonfatal = ( + # This should succeed if we've correctly saved it as A-1-1.gpkg.tar, not + # A-1-2.gpkg.tar, and then also try to unpack the right filename, but + # we defer checking the exit code to get a better error if the binpkg + # was downloaded with the wrong filename. + emerge_cmd + + ( + "--oneshot", + "--verbose", + "--getbinpkgonly", + "dev-libs/A", + ), + ) + test_commands_final = ( + # Check whether the downloaded binpkg in PKGDIR has the correct + # filename (-1) or an unnecessarily-incremented one (-2). + ( + lambda: os.path.exists( + os.path.join( + bindb.bintree.pkgdir, "dev-libs", "A", "A-1-1.gpkg.tar" + ) + ), + ), + ) + + fake_bin = os.path.join(eprefix, "bin") + portage_tmpdir = os.path.join(eprefix, "var", "tmp", "portage") + + path = settings.get("PATH") + if path is not None and not path.strip(): + path = None + if path is None: + path = "" + else: + path = ":" + path + path = fake_bin + path + + pythonpath = os.environ.get("PYTHONPATH") + if pythonpath is not None and not pythonpath.strip(): + pythonpath = None + if pythonpath is not None and pythonpath.split(":")[0] == PORTAGE_PYM_PATH: + pass + else: + if pythonpath is None: + pythonpath = "" + else: + pythonpath = ":" + pythonpath + pythonpath = PORTAGE_PYM_PATH + pythonpath + + env = { + "PORTAGE_OVERRIDE_EPREFIX": eprefix, + "PATH": path, + "PORTAGE_PYTHON": portage_python, + "PORTAGE_REPOSITORIES": settings.repositories.config_string(), + "PYTHONDONTWRITEBYTECODE": os.environ.get("PYTHONDONTWRITEBYTECODE", ""), + "PYTHONPATH": pythonpath, + "PORTAGE_INST_GID": str(os.getgid()), + "PORTAGE_INST_UID": str(os.getuid()), + "FEATURES": "-pkgdir-index-trusted", + } + + dirs = [ + playground.distdir, + fake_bin, + portage_tmpdir, + user_config_dir, + var_cache_edb, + ] + + true_symlinks = ["chown", "chgrp"] + + needed_binaries = { + "true": (find_binary("true"), True), + } + + def run_commands(test_commands, require_success=True): + all_successful = True + + for i, args in enumerate(test_commands): + if hasattr(args[0], "__call__"): + if require_success: + self.assertTrue(args[0](), f"callable at index {i} failed") + continue + + if isinstance(args[0], dict): + local_env = env.copy() + local_env.update(args[0]) + args = args[1:] + else: + local_env = env + + local_env["PORTAGE_BINHOST"] = f"file:///{tmppkgdir_suffix}" + proc = subprocess.Popen(args, env=local_env, stdout=stdout) + + if debug: + proc.wait() + else: + output = proc.stdout.readlines() + proc.wait() + proc.stdout.close() + if proc.returncode != os.EX_OK: + for line in output: + sys.stderr.write(_unicode_decode(line)) + + if all_successful and proc.returncode != os.EX_OK: + all_successful = False + + if require_success: + self.assertEqual( + os.EX_OK, proc.returncode, f"emerge failed with args {args}" + ) + + return all_successful + + try: + for d in dirs: + ensure_dirs(d) + for x in true_symlinks: + os.symlink(needed_binaries["true"][0], os.path.join(fake_bin, x)) + + with open(os.path.join(var_cache_edb, "counter"), "wb") as f: + f.write(b"100") + + if debug: + # The subprocess inherits both stdout and stderr, for + # debugging purposes. + stdout = None + else: + # The subprocess inherits stderr so that any warnings + # triggered by python -Wd will be visible. + stdout = subprocess.PIPE + + run_commands(test_commands) + deferred_success = run_commands(test_commands_nonfatal, False) + run_commands(test_commands_final) + + # Check the return value of test_commands_nonfatal later on so + # we can get a better error message from test_commands_final + # if possible. + self.assertTrue(deferred_success, f"{test_commands_nonfatal} failed") + finally: + playground.debug = False + playground.cleanup() + tmppkgdir.cleanup() diff --git a/lib/portage/tests/emerge/test_config_protect.py b/lib/portage/tests/emerge/test_config_protect.py index b60d0c495..59a2c23b9 100644 --- a/lib/portage/tests/emerge/test_config_protect.py +++ b/lib/portage/tests/emerge/test_config_protect.py @@ -1,8 +1,8 @@ -# Copyright 2014-2015 Gentoo Foundation +# Copyright 2014-2015, 2023 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -import io from functools import partial +import shlex import shutil import stat import subprocess @@ -16,7 +16,7 @@ from portage.const import BASH_BINARY, PORTAGE_PYM_PATH from portage.process import find_binary from portage.tests import TestCase from portage.tests.resolver.ResolverPlayground import ResolverPlayground -from portage.util import ensure_dirs, find_updated_config_files, shlex_split +from portage.util import ensure_dirs, find_updated_config_files class ConfigProtectTestCase(TestCase): @@ -113,10 +113,15 @@ src_install() { portage_python, "-b", "-Wd", - os.path.join(self.sbindir, "dispatch-conf"), + os.path.join(str(self.sbindir), "dispatch-conf"), ) - emerge_cmd = (portage_python, "-b", "-Wd", os.path.join(self.bindir, "emerge")) - etc_update_cmd = (BASH_BINARY, os.path.join(self.sbindir, "etc-update")) + emerge_cmd = ( + portage_python, + "-b", + "-Wd", + os.path.join(str(self.bindir), "emerge"), + ) + etc_update_cmd = (BASH_BINARY, os.path.join(str(self.sbindir), "etc-update")) etc_update_auto = etc_update_cmd + ( "--automode", "-5", @@ -129,7 +134,7 @@ src_install() { path = os.path.join(dir_path, name) st = os.lstat(path) if stat.S_ISREG(st.st_mode): - with io.open(path, mode="a", encoding=_encodings["stdio"]) as f: + with open(path, mode="a", encoding=_encodings["stdio"]) as f: f.write("modified at %d\n" % time.time()) elif stat.S_ISLNK(st.st_mode): old_dest = os.readlink(path) @@ -142,7 +147,7 @@ src_install() { sum( len(x[1]) for x in find_updated_config_files( - eroot, shlex_split(config_protect) + eroot, shlex.split(config_protect) ) ), ) @@ -187,7 +192,7 @@ src_install() { fake_bin = os.path.join(eprefix, "bin") portage_tmpdir = os.path.join(eprefix, "var", "tmp", "portage") - path = os.environ.get("PATH") + path = settings.get("PATH") if path is not None and not path.strip(): path = None if path is None: @@ -218,8 +223,8 @@ src_install() { "INFODIR": "", "INFOPATH": "", "PATH": path, - "PORTAGE_INST_GID": str(portage.data.portage_gid), - "PORTAGE_INST_UID": str(portage.data.portage_uid), + "PORTAGE_INST_GID": str(os.getgid()), # str(portage.data.portage_gid), + "PORTAGE_INST_UID": str(os.getuid()), # str(portage.data.portage_uid), "PORTAGE_PYTHON": portage_python, "PORTAGE_REPOSITORIES": settings.repositories.config_string(), "PORTAGE_TMPDIR": portage_tmpdir, @@ -248,7 +253,8 @@ src_install() { os.symlink(true_binary, os.path.join(fake_bin, x)) for x in etc_symlinks: os.symlink( - os.path.join(self.cnf_etc_path, x), os.path.join(eprefix, "etc", x) + os.path.join(str(self.cnf_etc_path), x), + os.path.join(eprefix, "etc", x), ) with open(os.path.join(var_cache_edb, "counter"), "wb") as f: f.write(b"100") @@ -263,7 +269,6 @@ src_install() { stdout = subprocess.PIPE for args in test_commands: - if hasattr(args, "__call__"): args() continue @@ -288,7 +293,7 @@ src_install() { sys.stderr.write(_unicode_decode(line)) self.assertEqual( - os.EX_OK, proc.returncode, "emerge failed with args %s" % (args,) + os.EX_OK, proc.returncode, f"emerge failed with args {args}" ) finally: playground.cleanup() diff --git a/lib/portage/tests/emerge/test_emerge_blocker_file_collision.py b/lib/portage/tests/emerge/test_emerge_blocker_file_collision.py index 785bf50cb..1eb7da79f 100644 --- a/lib/portage/tests/emerge/test_emerge_blocker_file_collision.py +++ b/lib/portage/tests/emerge/test_emerge_blocker_file_collision.py @@ -1,4 +1,4 @@ -# Copyright 2016 Gentoo Foundation +# Copyright 2016, 2023 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import subprocess @@ -16,7 +16,6 @@ from portage.util import ensure_dirs class BlockerFileCollisionEmergeTestCase(TestCase): def testBlockerFileCollision(self): - debug = False install_something = """ @@ -51,7 +50,12 @@ src_install() { user_config_dir = os.path.join(eprefix, USER_CONFIG_PATH) portage_python = portage._python_interpreter - emerge_cmd = (portage_python, "-b", "-Wd", os.path.join(self.bindir, "emerge")) + emerge_cmd = ( + portage_python, + "-b", + "-Wd", + os.path.join(str(self.bindir), "emerge"), + ) file_collision = os.path.join(eroot, "usr/lib/file-collision") @@ -94,7 +98,7 @@ src_install() { portage_tmpdir = os.path.join(eprefix, "var", "tmp", "portage") profile_path = settings.profile_path - path = os.environ.get("PATH") + path = settings.get("PATH") if path is not None and not path.strip(): path = None if path is None: @@ -122,6 +126,8 @@ src_install() { "PORTAGE_REPOSITORIES": settings.repositories.config_string(), "PYTHONDONTWRITEBYTECODE": os.environ.get("PYTHONDONTWRITEBYTECODE", ""), "PYTHONPATH": pythonpath, + "PORTAGE_INST_GID": str(os.getgid()), + "PORTAGE_INST_UID": str(os.getuid()), } if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: @@ -160,9 +166,8 @@ src_install() { stdout = subprocess.PIPE for i, args in enumerate(test_commands): - if hasattr(args[0], "__call__"): - self.assertTrue(args[0](), "callable at index %s failed" % (i,)) + self.assertTrue(args[0](), f"callable at index {i} failed") continue if isinstance(args[0], dict): @@ -185,7 +190,7 @@ src_install() { sys.stderr.write(_unicode_decode(line)) self.assertEqual( - os.EX_OK, proc.returncode, "emerge failed with args %s" % (args,) + os.EX_OK, proc.returncode, f"emerge failed with args {args}" ) finally: playground.debug = False diff --git a/lib/portage/tests/emerge/test_emerge_slot_abi.py b/lib/portage/tests/emerge/test_emerge_slot_abi.py index 3c3a8b582..c1a8fe894 100644 --- a/lib/portage/tests/emerge/test_emerge_slot_abi.py +++ b/lib/portage/tests/emerge/test_emerge_slot_abi.py @@ -1,4 +1,4 @@ -# Copyright 2012-2019 Gentoo Authors +# Copyright 2012-2019, 2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import subprocess @@ -16,7 +16,6 @@ from portage.util import ensure_dirs class SlotAbiEmergeTestCase(TestCase): def testSlotAbiEmerge(self): - debug = False ebuilds = { @@ -55,8 +54,18 @@ class SlotAbiEmergeTestCase(TestCase): package_mask_path = os.path.join(user_config_dir, "package.mask") portage_python = portage._python_interpreter - ebuild_cmd = (portage_python, "-b", "-Wd", os.path.join(self.bindir, "ebuild")) - emerge_cmd = (portage_python, "-b", "-Wd", os.path.join(self.bindir, "emerge")) + ebuild_cmd = ( + portage_python, + "-b", + "-Wd", + os.path.join(str(self.bindir), "ebuild"), + ) + emerge_cmd = ( + portage_python, + "-b", + "-Wd", + os.path.join(str(self.bindir), "emerge"), + ) test_ebuild = portdb.findname("dev-libs/dbus-glib-0.98") self.assertFalse(test_ebuild is None) @@ -102,7 +111,7 @@ class SlotAbiEmergeTestCase(TestCase): portage_tmpdir = os.path.join(eprefix, "var", "tmp", "portage") profile_path = settings.profile_path - path = os.environ.get("PATH") + path = settings.get("PATH") if path is not None and not path.strip(): path = None if path is None: @@ -130,6 +139,8 @@ class SlotAbiEmergeTestCase(TestCase): "PORTAGE_REPOSITORIES": settings.repositories.config_string(), "PYTHONDONTWRITEBYTECODE": os.environ.get("PYTHONDONTWRITEBYTECODE", ""), "PYTHONPATH": pythonpath, + "PORTAGE_INST_GID": str(os.getgid()), + "PORTAGE_INST_UID": str(os.getuid()), } if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: @@ -162,9 +173,8 @@ class SlotAbiEmergeTestCase(TestCase): stdout = subprocess.PIPE for i, args in enumerate(test_commands): - if hasattr(args[0], "__call__"): - self.assertTrue(args[0](), "callable at index %s failed" % (i,)) + self.assertTrue(args[0](), f"callable at index {i} failed") continue proc = subprocess.Popen(args, env=env, stdout=stdout) @@ -180,7 +190,7 @@ class SlotAbiEmergeTestCase(TestCase): sys.stderr.write(_unicode_decode(line)) self.assertEqual( - os.EX_OK, proc.returncode, "emerge failed with args %s" % (args,) + os.EX_OK, proc.returncode, f"emerge failed with args {args}" ) finally: playground.cleanup() diff --git a/lib/portage/tests/emerge/test_libc_dep_inject.py b/lib/portage/tests/emerge/test_libc_dep_inject.py new file mode 100644 index 000000000..933affcd7 --- /dev/null +++ b/lib/portage/tests/emerge/test_libc_dep_inject.py @@ -0,0 +1,552 @@ +# Copyright 2016-2023 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +import subprocess +import sys +import textwrap + +import portage +from portage import os +from portage import _unicode_decode +from portage.const import PORTAGE_PYM_PATH, USER_CONFIG_PATH +from portage.process import find_binary +from portage.tests import TestCase +from portage.util import ensure_dirs + +from portage.tests.resolver.ResolverPlayground import ( + ResolverPlayground, + ResolverPlaygroundTestCase, +) + + +class LibcDepInjectEmergeTestCase(TestCase): + def testLibcDepInjection(self): + """ + Test whether the implicit libc dependency injection (bug #913628) + is correctly added for only ebuilds installing an ELF binary. + + Based on BlockerFileCollisionEmergeTestCase. + """ + debug = False + + install_elf = textwrap.dedent( + """ + S="${WORKDIR}" + + src_install() { + insinto /usr/bin + # We need an ELF binary for the injection to trigger, so + # use ${BASH} given we know it must be around for running ebuilds. + cp "${BASH}" "${ED}"/usr/bin/${PN} || die + } + """ + ) + + ebuilds = { + "sys-libs/glibc-2.38": { + "EAPI": "8", + "MISC_CONTENT": install_elf, + }, + "virtual/libc-1": { + "EAPI": "8", + "RDEPEND": "sys-libs/glibc", + }, + "dev-libs/A-1": { + "EAPI": "8", + "MISC_CONTENT": install_elf, + }, + "dev-libs/B-1": { + "EAPI": "8", + }, + "dev-libs/C-1": { + "EAPI": "8", + "MISC_CONTENT": install_elf, + }, + "dev-libs/D-1": { + "EAPI": "8", + }, + "dev-libs/E-1": { + "EAPI": "8", + "RDEPEND": ">=dev-libs/D-1", + "MISC_CONTENT": install_elf, + }, + } + + world = ("dev-libs/A",) + + playground = ResolverPlayground(ebuilds=ebuilds, world=world, debug=debug) + settings = playground.settings + eprefix = settings["EPREFIX"] + eroot = settings["EROOT"] + var_cache_edb = os.path.join(eprefix, "var", "cache", "edb") + user_config_dir = os.path.join(eprefix, USER_CONFIG_PATH) + + portage_python = portage._python_interpreter + emerge_cmd = ( + portage_python, + "-b", + "-Wd", + os.path.join(str(self.bindir), "emerge"), + ) + + test_commands = ( + # If we install a package with an ELF but no libc provider is installed, + # make sure we don't inject anything (we don't want to have some bare RDEPEND with + # literally "[]"). + emerge_cmd + + ( + "--oneshot", + "dev-libs/C", + ), + ( + lambda: not portage.util.grablines( + os.path.join( + eprefix, "var", "db", "pkg", "dev-libs", "C-1", "RDEPEND" + ) + ), + ), + # (We need sys-libs/glibc pulled in and virtual/libc installed) + emerge_cmd + + ( + "--oneshot", + "virtual/libc", + ), + # A package NOT installing an ELF binary shouldn't have an injected libc dep + # Let's check the virtual/libc one as we already have to merge it to pull in + # sys-libs/glibc, but we'll do a better check after too. + ( + lambda: ">=sys-libs/glibc-2.38\n" + not in portage.util.grablines( + os.path.join( + eprefix, "var", "db", "pkg", "virtual", "libc-1", "RDEPEND" + ) + ), + ), + # A package NOT installing an ELF binary shouldn't have an injected libc dep + emerge_cmd + + ( + "--oneshot", + "dev-libs/B", + ), + ( + lambda: not portage.util.grablines( + os.path.join( + eprefix, "var", "db", "pkg", "dev-libs", "B-1", "RDEPEND" + ) + ), + ), + # A package installing an ELF binary should have an injected libc dep + emerge_cmd + + ( + "--oneshot", + "dev-libs/A", + ), + (lambda: os.path.exists(os.path.join(eroot, "usr/bin/A")),), + ( + lambda: ">=sys-libs/glibc-2.38\n" + in portage.util.grablines( + os.path.join( + eprefix, "var", "db", "pkg", "dev-libs", "A-1", "RDEPEND" + ) + ), + ), + # Install glibc again because earlier, no libc was installed, so the injection + # wouldn't have fired even if the "are we libc?" check was broken. + emerge_cmd + + ( + "--oneshot", + "sys-libs/glibc", + ), + # We don't want the libc (sys-libs/glibc is the provider here) to have an injected dep on itself + ( + lambda: ">=sys-libs/glibc-2.38\n" + not in portage.util.grablines( + os.path.join( + eprefix, "var", "db", "pkg", "sys-libs", "glibc-2.38", "RDEPEND" + ) + ), + ), + # Make sure we append to, not clobber, RDEPEND + emerge_cmd + + ( + "--oneshot", + "dev-libs/E", + ), + ( + lambda: [">=dev-libs/D-1 >=sys-libs/glibc-2.38\n"] + == portage.util.grablines( + os.path.join( + eprefix, "var", "db", "pkg", "dev-libs", "E-1", "RDEPEND" + ) + ), + ), + ) + + fake_bin = os.path.join(eprefix, "bin") + portage_tmpdir = os.path.join(eprefix, "var", "tmp", "portage") + profile_path = settings.profile_path + + path = settings.get("PATH") + if path is not None and not path.strip(): + path = None + if path is None: + path = "" + else: + path = ":" + path + path = fake_bin + path + + pythonpath = os.environ.get("PYTHONPATH") + if pythonpath is not None and not pythonpath.strip(): + pythonpath = None + if pythonpath is not None and pythonpath.split(":")[0] == PORTAGE_PYM_PATH: + pass + else: + if pythonpath is None: + pythonpath = "" + else: + pythonpath = ":" + pythonpath + pythonpath = PORTAGE_PYM_PATH + pythonpath + + env = { + "PORTAGE_OVERRIDE_EPREFIX": eprefix, + "PATH": path, + "PORTAGE_PYTHON": portage_python, + "PORTAGE_REPOSITORIES": settings.repositories.config_string(), + "PYTHONDONTWRITEBYTECODE": os.environ.get("PYTHONDONTWRITEBYTECODE", ""), + "PYTHONPATH": pythonpath, + "PORTAGE_INST_GID": str(os.getgid()), + "PORTAGE_INST_UID": str(os.getuid()), + "FEATURES": "-qa-unresolved-soname-deps -preserve-libs -merge-sync", + } + + dirs = [ + playground.distdir, + fake_bin, + portage_tmpdir, + user_config_dir, + var_cache_edb, + ] + + true_symlinks = ["chown", "chgrp"] + + # We don't want to make pax-utils a hard-requirement for tests, + # so if it's not found, skip the test rather than FAIL it. + needed_binaries = { + "true": (find_binary("true"), True), + "scanelf": (find_binary("scanelf"), False), + "find": (find_binary("find"), True), + } + + for name, (path, mandatory) in needed_binaries.items(): + found = path is not None + + if not found: + if mandatory: + self.assertIsNotNone(path, f"command {name} not found") + else: + self.skipTest(f"{name} not found") + + try: + for d in dirs: + ensure_dirs(d) + for x in true_symlinks: + os.symlink(needed_binaries["true"][0], os.path.join(fake_bin, x)) + + # We need scanelf, find for the ELF parts (creating NEEDED) + os.symlink(needed_binaries["scanelf"][0], os.path.join(fake_bin, "scanelf")) + os.symlink(needed_binaries["find"][0], os.path.join(fake_bin, "find")) + + with open(os.path.join(var_cache_edb, "counter"), "wb") as f: + f.write(b"100") + with open(os.path.join(profile_path, "packages"), "w") as f: + f.write("*virtual/libc") + + if debug: + # The subprocess inherits both stdout and stderr, for + # debugging purposes. + stdout = None + else: + # The subprocess inherits stderr so that any warnings + # triggered by python -Wd will be visible. + stdout = subprocess.PIPE + + for i, args in enumerate(test_commands): + if hasattr(args[0], "__call__"): + self.assertTrue(args[0](), f"callable at index {i} failed") + continue + + if isinstance(args[0], dict): + local_env = env.copy() + local_env.update(args[0]) + args = args[1:] + else: + local_env = env + + proc = subprocess.Popen(args, env=local_env, stdout=stdout) + + if debug: + proc.wait() + else: + output = proc.stdout.readlines() + proc.wait() + proc.stdout.close() + if proc.returncode != os.EX_OK: + for line in output: + sys.stderr.write(_unicode_decode(line)) + + self.assertEqual( + os.EX_OK, proc.returncode, f"emerge failed with args {args}" + ) + + # Check that dev-libs/A doesn't get re-emerged via --changed-deps + # after injecting the libc dep. We want to suppress the injected + # dep in the changed-deps comparisons. + k = ResolverPlaygroundTestCase( + ["@world"], + options={ + "--changed-deps": True, + "--deep": True, + "--update": True, + "--verbose": True, + }, + success=True, + mergelist=[], + ) + playground.run_TestCase(k) + self.assertEqual(k.test_success, True, k.fail_msg) + finally: + playground.debug = False + playground.cleanup() + + def testBinpkgLibcDepInjection(self): + """ + Test whether the implicit libc dependency injection (bug #913628) + correctly forces an upgrade to a newer glibc before merging a binpkg + built against it. + + Based on BlockerFileCollisionEmergeTestCase. + """ + debug = False + + install_elf = textwrap.dedent( + """ + S="${WORKDIR}" + + src_install() { + insinto /usr/bin + # We need an ELF binary for the injection to trigger, so + # use ${BASH} given we know it must be around for running ebuilds. + cp "${BASH}" "${ED}"/usr/bin/${PN} || die + } + """ + ) + + ebuilds = { + "sys-libs/glibc-2.37": { + "EAPI": "8", + "MISC_CONTENT": install_elf, + }, + "sys-libs/glibc-2.38": { + "EAPI": "8", + "MISC_CONTENT": install_elf, + }, + "virtual/libc-1": { + "EAPI": "8", + "RDEPEND": "sys-libs/glibc", + }, + "dev-libs/A-1": { + "EAPI": "8", + "MISC_CONTENT": install_elf, + }, + "dev-libs/B-1": { + "EAPI": "8", + }, + "dev-libs/C-1": { + "EAPI": "8", + "MISC_CONTENT": install_elf, + }, + } + + playground = ResolverPlayground(ebuilds=ebuilds, debug=debug) + settings = playground.settings + eprefix = settings["EPREFIX"] + eroot = settings["EROOT"] + var_cache_edb = os.path.join(eprefix, "var", "cache", "edb") + user_config_dir = os.path.join(eprefix, USER_CONFIG_PATH) + + portage_python = portage._python_interpreter + emerge_cmd = ( + portage_python, + "-b", + "-Wd", + os.path.join(str(self.bindir), "emerge"), + ) + + test_commands = ( + # (We need sys-libs/glibc pulled in and virtual/libc installed) + emerge_cmd + + ( + "--oneshot", + "virtual/libc", + ), + # A package installing an ELF binary should have an injected libc dep + emerge_cmd + + ( + "--oneshot", + "dev-libs/A", + ), + (lambda: os.path.exists(os.path.join(eroot, "usr/bin/A")),), + ( + lambda: ">=sys-libs/glibc-2.38\n" + in portage.util.grablines( + os.path.join( + eprefix, "var", "db", "pkg", "dev-libs", "A-1", "RDEPEND" + ) + ), + ), + # Downgrade glibc to a version (2.37) older than the version + # that dev-libs/A's binpkg was built against (2.38). Below, + # we check that it pulls in a newer glibc via a ResolverPlayground + # testcase. + emerge_cmd + + ( + "--oneshot", + "--nodeps", + "<sys-libs/glibc-2.38", + ), + ) + + fake_bin = os.path.join(eprefix, "bin") + portage_tmpdir = os.path.join(eprefix, "var", "tmp", "portage") + profile_path = settings.profile_path + + path = settings.get("PATH") + if path is not None and not path.strip(): + path = None + if path is None: + path = "" + else: + path = ":" + path + path = fake_bin + path + + pythonpath = os.environ.get("PYTHONPATH") + if pythonpath is not None and not pythonpath.strip(): + pythonpath = None + if pythonpath is not None and pythonpath.split(":")[0] == PORTAGE_PYM_PATH: + pass + else: + if pythonpath is None: + pythonpath = "" + else: + pythonpath = ":" + pythonpath + pythonpath = PORTAGE_PYM_PATH + pythonpath + + env = { + "PORTAGE_OVERRIDE_EPREFIX": eprefix, + "PATH": path, + "PORTAGE_PYTHON": portage_python, + "PORTAGE_REPOSITORIES": settings.repositories.config_string(), + "PYTHONDONTWRITEBYTECODE": os.environ.get("PYTHONDONTWRITEBYTECODE", ""), + "PYTHONPATH": pythonpath, + "PORTAGE_INST_GID": str(os.getgid()), + "PORTAGE_INST_UID": str(os.getuid()), + "FEATURES": "buildpkg", + } + + dirs = [ + playground.distdir, + fake_bin, + portage_tmpdir, + user_config_dir, + var_cache_edb, + ] + + true_symlinks = ["chown", "chgrp"] + + # We don't want to make pax-utils a hard-requirement for tests, + # so if it's not found, skip the test rather than FAIL it. + needed_binaries = { + "true": (find_binary("true"), True), + "scanelf": (find_binary("scanelf"), False), + "find": (find_binary("find"), True), + } + + for name, (path, mandatory) in needed_binaries.items(): + found = path is not None + + if not found: + if mandatory: + self.assertIsNotNone(path, f"command {name} not found") + else: + self.skipTest(f"{name} not found") + + try: + for d in dirs: + ensure_dirs(d) + for x in true_symlinks: + os.symlink(needed_binaries["true"][0], os.path.join(fake_bin, x)) + + # We need scanelf, find for the ELF parts (creating NEEDED) + os.symlink(needed_binaries["scanelf"][0], os.path.join(fake_bin, "scanelf")) + os.symlink(needed_binaries["find"][0], os.path.join(fake_bin, "find")) + + with open(os.path.join(var_cache_edb, "counter"), "wb") as f: + f.write(b"100") + with open(os.path.join(profile_path, "packages"), "w") as f: + f.write("*virtual/libc") + + if debug: + # The subprocess inherits both stdout and stderr, for + # debugging purposes. + stdout = None + else: + # The subprocess inherits stderr so that any warnings + # triggered by python -Wd will be visible. + stdout = subprocess.PIPE + + for i, args in enumerate(test_commands): + if hasattr(args[0], "__call__"): + self.assertTrue(args[0](), f"callable at index {i} failed") + continue + + if isinstance(args[0], dict): + local_env = env.copy() + local_env.update(args[0]) + args = args[1:] + else: + local_env = env + + proc = subprocess.Popen(args, env=local_env, stdout=stdout) + + if debug: + proc.wait() + else: + output = proc.stdout.readlines() + proc.wait() + proc.stdout.close() + if proc.returncode != os.EX_OK: + for line in output: + sys.stderr.write(_unicode_decode(line)) + + self.assertEqual( + os.EX_OK, proc.returncode, f"emerge failed with args {args}" + ) + + # Now check that glibc gets upgraded to the right version + # for the binpkg first after we downgraded it earlier, before + # merging the dev-libs/A binpkg which needs 2.38. + k = ResolverPlaygroundTestCase( + ["dev-libs/A"], + options={ + "--usepkgonly": True, + "--verbose": True, + }, + success=True, + mergelist=["[binary]sys-libs/glibc-2.38-1", "[binary]dev-libs/A-1-1"], + ) + playground.run_TestCase(k) + self.assertEqual(k.test_success, True, k.fail_msg) + + finally: + playground.debug = False + playground.cleanup() diff --git a/lib/portage/tests/emerge/test_simple.py b/lib/portage/tests/emerge/test_simple.py deleted file mode 100644 index 3a8bf3764..000000000 --- a/lib/portage/tests/emerge/test_simple.py +++ /dev/null @@ -1,704 +0,0 @@ -# Copyright 2011-2021 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -import argparse -import subprocess - -import portage -from portage import shutil, os -from portage.const import ( - BASH_BINARY, - BINREPOS_CONF_FILE, - PORTAGE_PYM_PATH, - USER_CONFIG_PATH, -) -from portage.cache.mappings import Mapping -from portage.process import find_binary -from portage.tests import TestCase -from portage.tests.resolver.ResolverPlayground import ResolverPlayground -from portage.tests.util.test_socks5 import AsyncHTTPServer -from portage.util import ensure_dirs, find_updated_config_files, shlex_split -from portage.util.futures import asyncio - - -class BinhostContentMap(Mapping): - def __init__(self, remote_path, local_path): - self._remote_path = remote_path - self._local_path = local_path - - def __getitem__(self, request_path): - safe_path = os.path.normpath(request_path) - if not safe_path.startswith(self._remote_path + "/"): - raise KeyError(request_path) - local_path = os.path.join( - self._local_path, safe_path[len(self._remote_path) + 1 :] - ) - try: - with open(local_path, "rb") as f: - return f.read() - except EnvironmentError: - raise KeyError(request_path) - - -class SimpleEmergeTestCase(TestCase): - def _have_python_xml(self): - try: - __import__("xml.etree.ElementTree") - __import__("xml.parsers.expat").parsers.expat.ExpatError - except (AttributeError, ImportError): - return False - return True - - def testSimple(self): - - debug = False - - install_something = """ -S="${WORKDIR}" - -pkg_pretend() { - einfo "called pkg_pretend for $CATEGORY/$PF" -} - -src_install() { - einfo "installing something..." - insinto /usr/lib/${P} - echo "blah blah blah" > "${T}"/regular-file - doins "${T}"/regular-file - dosym regular-file /usr/lib/${P}/symlink || die - - # Test CONFIG_PROTECT - insinto /etc - newins "${T}"/regular-file ${PN}-${SLOT%/*} - - # Test code for bug #381629, using a copyright symbol encoded with latin-1. - # We use $(printf "\\xa9") rather than $'\\xa9', since printf apparently - # works in any case, while $'\\xa9' transforms to \\xef\\xbf\\xbd under - # some conditions. TODO: Find out why it transforms to \\xef\\xbf\\xbd when - # running tests for Python 3.2 (even though it's bash that is ultimately - # responsible for performing the transformation). - local latin_1_dir=/usr/lib/${P}/latin-1-$(printf "\\xa9")-directory - insinto "${latin_1_dir}" - echo "blah blah blah" > "${T}"/latin-1-$(printf "\\xa9")-regular-file || die - doins "${T}"/latin-1-$(printf "\\xa9")-regular-file - dosym latin-1-$(printf "\\xa9")-regular-file ${latin_1_dir}/latin-1-$(printf "\\xa9")-symlink || die - - call_has_and_best_version -} - -pkg_config() { - einfo "called pkg_config for $CATEGORY/$PF" -} - -pkg_info() { - einfo "called pkg_info for $CATEGORY/$PF" -} - -pkg_preinst() { - if ! ___eapi_best_version_and_has_version_support_-b_-d_-r; then - # The BROOT variable is unset during pkg_* phases for EAPI 7, - # therefore best/has_version -b is expected to fail if we attempt - # to call it for EAPI 7 here. - call_has_and_best_version - fi -} - -call_has_and_best_version() { - local root_arg - if ___eapi_best_version_and_has_version_support_-b_-d_-r; then - root_arg="-b" - else - root_arg="--host-root" - fi - einfo "called ${EBUILD_PHASE_FUNC} for $CATEGORY/$PF" - einfo "EPREFIX=${EPREFIX}" - einfo "PORTAGE_OVERRIDE_EPREFIX=${PORTAGE_OVERRIDE_EPREFIX}" - einfo "ROOT=${ROOT}" - einfo "EROOT=${EROOT}" - einfo "SYSROOT=${SYSROOT}" - einfo "ESYSROOT=${ESYSROOT}" - einfo "BROOT=${BROOT}" - # Test that has_version and best_version work correctly with - # prefix (involves internal ROOT -> EROOT calculation in order - # to support ROOT override via the environment with EAPIs 3 - # and later which support prefix). - if has_version $CATEGORY/$PN:$SLOT ; then - einfo "has_version detects an installed instance of $CATEGORY/$PN:$SLOT" - einfo "best_version reports that the installed instance is $(best_version $CATEGORY/$PN:$SLOT)" - else - einfo "has_version does not detect an installed instance of $CATEGORY/$PN:$SLOT" - fi - if [[ ${EPREFIX} != ${PORTAGE_OVERRIDE_EPREFIX} ]] ; then - if has_version ${root_arg} $CATEGORY/$PN:$SLOT ; then - einfo "has_version ${root_arg} detects an installed instance of $CATEGORY/$PN:$SLOT" - einfo "best_version ${root_arg} reports that the installed instance is $(best_version ${root_arg} $CATEGORY/$PN:$SLOT)" - else - einfo "has_version ${root_arg} does not detect an installed instance of $CATEGORY/$PN:$SLOT" - fi - fi -} - -""" - - ebuilds = { - "dev-libs/A-1": { - "EAPI": "5", - "IUSE": "+flag", - "KEYWORDS": "x86", - "LICENSE": "GPL-2", - "MISC_CONTENT": install_something, - "RDEPEND": "flag? ( dev-libs/B[flag] )", - }, - "dev-libs/B-1": { - "EAPI": "5", - "IUSE": "+flag", - "KEYWORDS": "x86", - "LICENSE": "GPL-2", - "MISC_CONTENT": install_something, - }, - "dev-libs/C-1": { - "EAPI": "7", - "KEYWORDS": "~x86", - "RDEPEND": "dev-libs/D[flag]", - "MISC_CONTENT": install_something, - }, - "dev-libs/D-1": { - "EAPI": "7", - "KEYWORDS": "~x86", - "IUSE": "flag", - "MISC_CONTENT": install_something, - }, - "virtual/foo-0": { - "EAPI": "5", - "KEYWORDS": "x86", - "LICENSE": "GPL-2", - }, - } - - installed = { - "dev-libs/A-1": { - "EAPI": "5", - "IUSE": "+flag", - "KEYWORDS": "x86", - "LICENSE": "GPL-2", - "RDEPEND": "flag? ( dev-libs/B[flag] )", - "USE": "flag", - }, - "dev-libs/B-1": { - "EAPI": "5", - "IUSE": "+flag", - "KEYWORDS": "x86", - "LICENSE": "GPL-2", - "USE": "flag", - }, - "dev-libs/depclean-me-1": { - "EAPI": "5", - "IUSE": "", - "KEYWORDS": "x86", - "LICENSE": "GPL-2", - "USE": "", - }, - "app-misc/depclean-me-1": { - "EAPI": "5", - "IUSE": "", - "KEYWORDS": "x86", - "LICENSE": "GPL-2", - "RDEPEND": "dev-libs/depclean-me", - "USE": "", - }, - } - - metadata_xml_files = ( - ( - "dev-libs/A", - { - "flags": "<flag name='flag'>Description of how USE='flag' affects this package</flag>", - }, - ), - ( - "dev-libs/B", - { - "flags": "<flag name='flag'>Description of how USE='flag' affects this package</flag>", - }, - ), - ) - - playground = ResolverPlayground( - ebuilds=ebuilds, installed=installed, debug=debug - ) - - loop = asyncio._wrap_loop() - loop.run_until_complete( - asyncio.ensure_future( - self._async_test_simple(playground, metadata_xml_files, loop=loop), - loop=loop, - ) - ) - - async def _async_test_simple(self, playground, metadata_xml_files, loop): - - debug = playground.debug - settings = playground.settings - eprefix = settings["EPREFIX"] - eroot = settings["EROOT"] - trees = playground.trees - portdb = trees[eroot]["porttree"].dbapi - test_repo_location = settings.repositories["test_repo"].location - var_cache_edb = os.path.join(eprefix, "var", "cache", "edb") - cachedir = os.path.join(var_cache_edb, "dep") - cachedir_pregen = os.path.join(test_repo_location, "metadata", "md5-cache") - - portage_python = portage._python_interpreter - dispatch_conf_cmd = ( - portage_python, - "-b", - "-Wd", - os.path.join(self.sbindir, "dispatch-conf"), - ) - ebuild_cmd = (portage_python, "-b", "-Wd", os.path.join(self.bindir, "ebuild")) - egencache_cmd = ( - portage_python, - "-b", - "-Wd", - os.path.join(self.bindir, "egencache"), - "--repo", - "test_repo", - "--repositories-configuration", - settings.repositories.config_string(), - ) - emerge_cmd = (portage_python, "-b", "-Wd", os.path.join(self.bindir, "emerge")) - emaint_cmd = (portage_python, "-b", "-Wd", os.path.join(self.sbindir, "emaint")) - env_update_cmd = ( - portage_python, - "-b", - "-Wd", - os.path.join(self.sbindir, "env-update"), - ) - etc_update_cmd = (BASH_BINARY, os.path.join(self.sbindir, "etc-update")) - fixpackages_cmd = ( - portage_python, - "-b", - "-Wd", - os.path.join(self.sbindir, "fixpackages"), - ) - portageq_cmd = ( - portage_python, - "-b", - "-Wd", - os.path.join(self.bindir, "portageq"), - ) - quickpkg_cmd = ( - portage_python, - "-b", - "-Wd", - os.path.join(self.bindir, "quickpkg"), - ) - regenworld_cmd = ( - portage_python, - "-b", - "-Wd", - os.path.join(self.sbindir, "regenworld"), - ) - - rm_binary = find_binary("rm") - self.assertEqual(rm_binary is None, False, "rm command not found") - rm_cmd = (rm_binary,) - - egencache_extra_args = [] - if self._have_python_xml(): - egencache_extra_args.append("--update-use-local-desc") - - test_ebuild = portdb.findname("dev-libs/A-1") - self.assertFalse(test_ebuild is None) - - cross_prefix = os.path.join(eprefix, "cross_prefix") - cross_root = os.path.join(eprefix, "cross_root") - cross_eroot = os.path.join(cross_root, eprefix.lstrip(os.sep)) - - binhost_dir = os.path.join(eprefix, "binhost") - binhost_address = "127.0.0.1" - binhost_remote_path = "/binhost" - binhost_server = AsyncHTTPServer( - binhost_address, BinhostContentMap(binhost_remote_path, binhost_dir), loop - ).__enter__() - binhost_uri = "http://{address}:{port}{path}".format( - address=binhost_address, - port=binhost_server.server_port, - path=binhost_remote_path, - ) - - test_commands = () - - if hasattr(argparse.ArgumentParser, "parse_intermixed_args"): - test_commands += ( - emerge_cmd + ("--oneshot", "dev-libs/A", "-v", "dev-libs/A"), - ) - - test_commands += ( - emerge_cmd - + ( - "--usepkgonly", - "--root", - cross_root, - "--quickpkg-direct=y", - "--quickpkg-direct-root", - "/", - "dev-libs/A", - ), - emerge_cmd - + ( - "--usepkgonly", - "--quickpkg-direct=y", - "--quickpkg-direct-root", - cross_root, - "dev-libs/A", - ), - env_update_cmd, - portageq_cmd - + ( - "envvar", - "-v", - "CONFIG_PROTECT", - "EROOT", - "PORTAGE_CONFIGROOT", - "PORTAGE_TMPDIR", - "USERLAND", - ), - etc_update_cmd, - dispatch_conf_cmd, - emerge_cmd + ("--version",), - emerge_cmd + ("--info",), - emerge_cmd + ("--info", "--verbose"), - emerge_cmd + ("--list-sets",), - emerge_cmd + ("--check-news",), - rm_cmd + ("-rf", cachedir), - rm_cmd + ("-rf", cachedir_pregen), - emerge_cmd + ("--regen",), - rm_cmd + ("-rf", cachedir), - ({"FEATURES": "metadata-transfer"},) + emerge_cmd + ("--regen",), - rm_cmd + ("-rf", cachedir), - ({"FEATURES": "metadata-transfer"},) + emerge_cmd + ("--regen",), - rm_cmd + ("-rf", cachedir), - egencache_cmd + ("--update",) + tuple(egencache_extra_args), - ({"FEATURES": "metadata-transfer"},) + emerge_cmd + ("--metadata",), - rm_cmd + ("-rf", cachedir), - ({"FEATURES": "metadata-transfer"},) + emerge_cmd + ("--metadata",), - emerge_cmd + ("--metadata",), - rm_cmd + ("-rf", cachedir), - emerge_cmd + ("--oneshot", "virtual/foo"), - lambda: self.assertFalse( - os.path.exists(os.path.join(pkgdir, "virtual", "foo", "foo-0-1.xpak")) - ), - ({"FEATURES": "unmerge-backup"},) - + emerge_cmd - + ("--unmerge", "virtual/foo"), - lambda: self.assertTrue( - os.path.exists(os.path.join(pkgdir, "virtual", "foo", "foo-0-1.xpak")) - ), - emerge_cmd + ("--pretend", "dev-libs/A"), - ebuild_cmd + (test_ebuild, "manifest", "clean", "package", "merge"), - emerge_cmd + ("--pretend", "--tree", "--complete-graph", "dev-libs/A"), - emerge_cmd + ("-p", "dev-libs/B"), - emerge_cmd + ("-p", "--newrepo", "dev-libs/B"), - emerge_cmd - + ( - "-B", - "dev-libs/B", - ), - emerge_cmd - + ( - "--oneshot", - "--usepkg", - "dev-libs/B", - ), - # trigger clean prior to pkg_pretend as in bug #390711 - ebuild_cmd + (test_ebuild, "unpack"), - emerge_cmd - + ( - "--oneshot", - "dev-libs/A", - ), - emerge_cmd - + ( - "--noreplace", - "dev-libs/A", - ), - emerge_cmd - + ( - "--config", - "dev-libs/A", - ), - emerge_cmd + ("--info", "dev-libs/A", "dev-libs/B"), - emerge_cmd + ("--pretend", "--depclean", "--verbose", "dev-libs/B"), - emerge_cmd - + ( - "--pretend", - "--depclean", - ), - emerge_cmd + ("--depclean",), - quickpkg_cmd - + ( - "--include-config", - "y", - "dev-libs/A", - ), - # Test bug #523684, where a file renamed or removed by the - # admin forces replacement files to be merged with config - # protection. - lambda: self.assertEqual( - 0, - len( - list( - find_updated_config_files( - eroot, shlex_split(settings["CONFIG_PROTECT"]) - ) - ) - ), - ), - lambda: os.unlink(os.path.join(eprefix, "etc", "A-0")), - emerge_cmd + ("--usepkgonly", "dev-libs/A"), - lambda: self.assertEqual( - 1, - len( - list( - find_updated_config_files( - eroot, shlex_split(settings["CONFIG_PROTECT"]) - ) - ) - ), - ), - emaint_cmd + ("--check", "all"), - emaint_cmd + ("--fix", "all"), - fixpackages_cmd, - regenworld_cmd, - portageq_cmd + ("match", eroot, "dev-libs/A"), - portageq_cmd + ("best_visible", eroot, "dev-libs/A"), - portageq_cmd + ("best_visible", eroot, "binary", "dev-libs/A"), - portageq_cmd + ("contents", eroot, "dev-libs/A-1"), - portageq_cmd - + ("metadata", eroot, "ebuild", "dev-libs/A-1", "EAPI", "IUSE", "RDEPEND"), - portageq_cmd - + ("metadata", eroot, "binary", "dev-libs/A-1", "EAPI", "USE", "RDEPEND"), - portageq_cmd - + ( - "metadata", - eroot, - "installed", - "dev-libs/A-1", - "EAPI", - "USE", - "RDEPEND", - ), - portageq_cmd + ("owners", eroot, eroot + "usr"), - emerge_cmd + ("-p", eroot + "usr"), - emerge_cmd + ("-p", "--unmerge", "-q", eroot + "usr"), - emerge_cmd + ("--unmerge", "--quiet", "dev-libs/A"), - emerge_cmd + ("-C", "--quiet", "dev-libs/B"), - # If EMERGE_DEFAULT_OPTS contains --autounmask=n, then --autounmask - # must be specified with --autounmask-continue. - ({"EMERGE_DEFAULT_OPTS": "--autounmask=n"},) - + emerge_cmd - + ( - "--autounmask", - "--autounmask-continue", - "dev-libs/C", - ), - # Verify that the above --autounmask-continue command caused - # USE=flag to be applied correctly to dev-libs/D. - portageq_cmd + ("match", eroot, "dev-libs/D[flag]"), - # Test cross-prefix usage, including chpathtool for binpkgs. - # EAPI 7 - ({"EPREFIX": cross_prefix},) + emerge_cmd + ("dev-libs/C",), - ({"EPREFIX": cross_prefix},) - + portageq_cmd - + ("has_version", cross_prefix, "dev-libs/C"), - ({"EPREFIX": cross_prefix},) - + portageq_cmd - + ("has_version", cross_prefix, "dev-libs/D"), - ({"ROOT": cross_root},) + emerge_cmd + ("dev-libs/D",), - portageq_cmd + ("has_version", cross_eroot, "dev-libs/D"), - # EAPI 5 - ({"EPREFIX": cross_prefix},) + emerge_cmd + ("--usepkgonly", "dev-libs/A"), - ({"EPREFIX": cross_prefix},) - + portageq_cmd - + ("has_version", cross_prefix, "dev-libs/A"), - ({"EPREFIX": cross_prefix},) - + portageq_cmd - + ("has_version", cross_prefix, "dev-libs/B"), - ({"EPREFIX": cross_prefix},) + emerge_cmd + ("-C", "--quiet", "dev-libs/B"), - ({"EPREFIX": cross_prefix},) + emerge_cmd + ("-C", "--quiet", "dev-libs/A"), - ({"EPREFIX": cross_prefix},) + emerge_cmd + ("dev-libs/A",), - ({"EPREFIX": cross_prefix},) - + portageq_cmd - + ("has_version", cross_prefix, "dev-libs/A"), - ({"EPREFIX": cross_prefix},) - + portageq_cmd - + ("has_version", cross_prefix, "dev-libs/B"), - # Test ROOT support - ({"ROOT": cross_root},) + emerge_cmd + ("dev-libs/B",), - portageq_cmd + ("has_version", cross_eroot, "dev-libs/B"), - ) - - # Test binhost support if FETCHCOMMAND is available. - binrepos_conf_file = os.path.join(os.sep, eprefix, BINREPOS_CONF_FILE) - with open(binrepos_conf_file, "wt") as f: - f.write("[test-binhost]\n") - f.write("sync-uri = {}\n".format(binhost_uri)) - fetchcommand = portage.util.shlex_split(playground.settings["FETCHCOMMAND"]) - fetch_bin = portage.process.find_binary(fetchcommand[0]) - if fetch_bin is not None: - test_commands = test_commands + ( - lambda: os.rename(pkgdir, binhost_dir), - emerge_cmd + ("-e", "--getbinpkgonly", "dev-libs/A"), - lambda: shutil.rmtree(pkgdir), - lambda: os.rename(binhost_dir, pkgdir), - # Remove binrepos.conf and test PORTAGE_BINHOST. - lambda: os.unlink(binrepos_conf_file), - lambda: os.rename(pkgdir, binhost_dir), - ({"PORTAGE_BINHOST": binhost_uri},) - + emerge_cmd - + ("-fe", "--getbinpkgonly", "dev-libs/A"), - lambda: shutil.rmtree(pkgdir), - lambda: os.rename(binhost_dir, pkgdir), - ) - - distdir = playground.distdir - pkgdir = playground.pkgdir - fake_bin = os.path.join(eprefix, "bin") - portage_tmpdir = os.path.join(eprefix, "var", "tmp", "portage") - profile_path = settings.profile_path - user_config_dir = os.path.join(os.sep, eprefix, USER_CONFIG_PATH) - - path = os.environ.get("PATH") - if path is not None and not path.strip(): - path = None - if path is None: - path = "" - else: - path = ":" + path - path = fake_bin + path - - pythonpath = os.environ.get("PYTHONPATH") - if pythonpath is not None and not pythonpath.strip(): - pythonpath = None - if pythonpath is not None and pythonpath.split(":")[0] == PORTAGE_PYM_PATH: - pass - else: - if pythonpath is None: - pythonpath = "" - else: - pythonpath = ":" + pythonpath - pythonpath = PORTAGE_PYM_PATH + pythonpath - - env = { - "PORTAGE_OVERRIDE_EPREFIX": eprefix, - "CLEAN_DELAY": "0", - "DISTDIR": distdir, - "EMERGE_WARNING_DELAY": "0", - "INFODIR": "", - "INFOPATH": "", - "PATH": path, - "PKGDIR": pkgdir, - "PORTAGE_INST_GID": str(portage.data.portage_gid), - "PORTAGE_INST_UID": str(portage.data.portage_uid), - "PORTAGE_PYTHON": portage_python, - "PORTAGE_REPOSITORIES": settings.repositories.config_string(), - "PORTAGE_TMPDIR": portage_tmpdir, - "PORTAGE_LOGDIR": portage_tmpdir, - "PYTHONDONTWRITEBYTECODE": os.environ.get("PYTHONDONTWRITEBYTECODE", ""), - "PYTHONPATH": pythonpath, - "__PORTAGE_TEST_PATH_OVERRIDE": fake_bin, - } - - if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: - env["__PORTAGE_TEST_HARDLINK_LOCKS"] = os.environ[ - "__PORTAGE_TEST_HARDLINK_LOCKS" - ] - - updates_dir = os.path.join(test_repo_location, "profiles", "updates") - dirs = [ - cachedir, - cachedir_pregen, - cross_eroot, - cross_prefix, - distdir, - fake_bin, - portage_tmpdir, - updates_dir, - user_config_dir, - var_cache_edb, - ] - etc_symlinks = ("dispatch-conf.conf", "etc-update.conf") - # Override things that may be unavailable, or may have portability - # issues when running tests in exotic environments. - # prepstrip - bug #447810 (bash read builtin EINTR problem) - true_symlinks = ["find", "prepstrip", "sed", "scanelf"] - true_binary = find_binary("true") - self.assertEqual(true_binary is None, False, "true command not found") - try: - for d in dirs: - ensure_dirs(d) - for x in true_symlinks: - os.symlink(true_binary, os.path.join(fake_bin, x)) - for x in etc_symlinks: - os.symlink( - os.path.join(self.cnf_etc_path, x), os.path.join(eprefix, "etc", x) - ) - with open(os.path.join(var_cache_edb, "counter"), "wb") as f: - f.write(b"100") - # non-empty system set keeps --depclean quiet - with open(os.path.join(profile_path, "packages"), "w") as f: - f.write("*dev-libs/token-system-pkg") - for cp, xml_data in metadata_xml_files: - with open( - os.path.join(test_repo_location, cp, "metadata.xml"), "w" - ) as f: - f.write(playground.metadata_xml_template % xml_data) - with open(os.path.join(updates_dir, "1Q-2010"), "w") as f: - f.write( - """ -slotmove =app-doc/pms-3 2 3 -move dev-util/git dev-vcs/git -""" - ) - - if debug: - # The subprocess inherits both stdout and stderr, for - # debugging purposes. - stdout = None - else: - # The subprocess inherits stderr so that any warnings - # triggered by python -Wd will be visible. - stdout = subprocess.PIPE - - for args in test_commands: - - if hasattr(args, "__call__"): - args() - continue - - if isinstance(args[0], dict): - local_env = env.copy() - local_env.update(args[0]) - args = args[1:] - else: - local_env = env - - proc = await asyncio.create_subprocess_exec( - *args, env=local_env, stderr=None, stdout=stdout - ) - - if debug: - await proc.wait() - else: - output, _err = await proc.communicate() - await proc.wait() - if proc.returncode != os.EX_OK: - portage.writemsg(output) - - self.assertEqual( - os.EX_OK, proc.returncode, "emerge failed with args %s" % (args,) - ) - finally: - binhost_server.__exit__(None, None, None) - playground.cleanup() |