diff options
Diffstat (limited to 'lib/portage/package/ebuild')
27 files changed, 925 insertions, 994 deletions
diff --git a/lib/portage/package/ebuild/_config/LicenseManager.py b/lib/portage/package/ebuild/_config/LicenseManager.py index c28cc89bf..90f7742e3 100644 --- a/lib/portage/package/ebuild/_config/LicenseManager.py +++ b/lib/portage/package/ebuild/_config/LicenseManager.py @@ -15,7 +15,6 @@ from portage.package.ebuild._config.helper import ordered_by_atom_specificity class LicenseManager: def __init__(self, license_group_locations, abs_user_config, user_config=True): - self._accept_license_str = None self._accept_license = None self._license_groups = {} diff --git a/lib/portage/package/ebuild/_config/LocationsManager.py b/lib/portage/package/ebuild/_config/LocationsManager.py index d65aac609..6c54b8056 100644 --- a/lib/portage/package/ebuild/_config/LocationsManager.py +++ b/lib/portage/package/ebuild/_config/LocationsManager.py @@ -3,7 +3,6 @@ __all__ = ("LocationsManager",) -import io import warnings import portage @@ -93,16 +92,12 @@ class LocationsManager: + os.sep ) - self.esysroot = self.sysroot.rstrip(os.sep) + self.eprefix + os.sep - # TODO: Set this via the constructor using # PORTAGE_OVERRIDE_EPREFIX. self.broot = portage.const.EPREFIX def load_profiles(self, repositories, known_repository_paths): - known_repository_paths = set( - os.path.realpath(x) for x in known_repository_paths - ) + known_repository_paths = {os.path.realpath(x) for x in known_repository_paths} known_repos = [] for x in known_repository_paths: @@ -165,7 +160,7 @@ class LocationsManager: _("!!! Unable to parse profile: '%s'\n") % self.profile_path, noiselevel=-1, ) - writemsg("!!! ParseError: %s\n" % str(e), noiselevel=-1) + writemsg(f"!!! ParseError: {str(e)}\n", noiselevel=-1) self.profiles = [] self.profiles_complex = [] @@ -226,14 +221,13 @@ class LocationsManager: eapi = eapi or "0" f = None try: - f = io.open( + f = open( _unicode_encode(eapi_file, encoding=_encodings["fs"], errors="strict"), - mode="r", encoding=_encodings["content"], errors="replace", ) eapi = f.readline().strip() - except IOError: + except OSError: pass else: if not eapi_is_supported(eapi): @@ -389,13 +383,13 @@ class LocationsManager: + os.path.sep ) - if self.sysroot != "/" and self.sysroot != self.target_root: + if self.sysroot != "/" and self.target_root == "/": writemsg( _( "!!! Error: SYSROOT (currently %s) must " - "equal / or ROOT (currently %s).\n" + "be set to / when ROOT is /.\n" ) - % (self.sysroot, self.target_root), + % self.sysroot, noiselevel=-1, ) raise InvalidLocation(self.sysroot) @@ -405,6 +399,15 @@ class LocationsManager: self.eroot = self.target_root.rstrip(os.sep) + self.eprefix + os.sep + # In a cross-prefix scenario where SYSROOT=/ and ROOT=/, assume we want + # ESYSROOT to point to the target prefix. + if self.sysroot == self.target_root: + self.esysroot = self.sysroot.rstrip(os.sep) + self.eprefix + os.sep + elif self.sysroot == "/": + self.esysroot = self.broot + os.sep + else: + self.esysroot = self.sysroot + self.global_config_path = GLOBAL_CONFIG_PATH if portage.const.EPREFIX: self.global_config_path = os.path.join( diff --git a/lib/portage/package/ebuild/_config/UseManager.py b/lib/portage/package/ebuild/_config/UseManager.py index a0fb0254a..3827ba27a 100644 --- a/lib/portage/package/ebuild/_config/UseManager.py +++ b/lib/portage/package/ebuild/_config/UseManager.py @@ -6,24 +6,19 @@ __all__ = ("UseManager",) from _emerge.Package import Package from portage import os from portage.dep import ( - Atom, dep_getrepo, dep_getslot, ExtendedAtomDict, remove_slot, _get_useflag_re, - _repo_separator, ) from portage.eapi import ( - eapi_has_use_aliases, eapi_supports_stable_use_forcing_and_masking, ) -from portage.exception import InvalidAtom from portage.localization import _ from portage.repository.config import allow_profile_repo_deps from portage.util import ( grabfile, - grabdict, grabdict_package, read_corresponding_eapi_file, stack_lists, @@ -46,12 +41,10 @@ class UseManager: # use.stable.mask _repo_usestablemask_dict # use.force _repo_useforce_dict # use.stable.force _repo_usestableforce_dict - # use.aliases _repo_usealiases_dict # package.use.mask _repo_pusemask_dict # package.use.stable.mask _repo_pusestablemask_dict # package.use.force _repo_puseforce_dict # package.use.stable.force _repo_pusestableforce_dict - # package.use.aliases _repo_pusealiases_dict # -------------------------------- # profiles # -------------------------------- @@ -158,11 +151,6 @@ class UseManager: "package.use", abs_user_config, user_config ) - self._repo_usealiases_dict = self._parse_repository_usealiases(repositories) - self._repo_pusealiases_dict = self._parse_repository_packageusealiases( - repositories - ) - self.repositories = repositories def _parse_file_to_tuple( @@ -409,111 +397,6 @@ class UseManager: for profile in locations ) - def _parse_repository_usealiases(self, repositories): - ret = {} - for repo in repositories.repos_with_profiles(): - file_name = os.path.join(repo.location, "profiles", "use.aliases") - eapi = read_corresponding_eapi_file(file_name, default=repo.eapi) - useflag_re = _get_useflag_re(eapi) - raw_file_dict = grabdict(file_name, recursive=True) - file_dict = {} - for real_flag, aliases in raw_file_dict.items(): - if useflag_re.match(real_flag) is None: - writemsg( - _("--- Invalid real USE flag in '%s': '%s'\n") - % (file_name, real_flag), - noiselevel=-1, - ) - else: - for alias in aliases: - if useflag_re.match(alias) is None: - writemsg( - _( - "--- Invalid USE flag alias for '%s' real USE flag in '%s': '%s'\n" - ) - % (real_flag, file_name, alias), - noiselevel=-1, - ) - else: - if any( - alias in v - for k, v in file_dict.items() - if k != real_flag - ): - writemsg( - _("--- Duplicated USE flag alias in '%s': '%s'\n") - % (file_name, alias), - noiselevel=-1, - ) - else: - file_dict.setdefault(real_flag, []).append(alias) - ret[repo.name] = file_dict - return ret - - def _parse_repository_packageusealiases(self, repositories): - ret = {} - for repo in repositories.repos_with_profiles(): - file_name = os.path.join(repo.location, "profiles", "package.use.aliases") - eapi = read_corresponding_eapi_file(file_name, default=repo.eapi) - useflag_re = _get_useflag_re(eapi) - lines = grabfile(file_name, recursive=True) - file_dict = {} - for line in lines: - elements = line.split() - atom = elements[0] - try: - atom = Atom(atom, eapi=eapi) - except InvalidAtom: - writemsg(_("--- Invalid atom in '%s': '%s'\n") % (file_name, atom)) - continue - if len(elements) == 1: - writemsg( - _("--- Missing real USE flag for '%s' in '%s'\n") - % (atom, file_name), - noiselevel=-1, - ) - continue - real_flag = elements[1] - if useflag_re.match(real_flag) is None: - writemsg( - _("--- Invalid real USE flag for '%s' in '%s': '%s'\n") - % (atom, file_name, real_flag), - noiselevel=-1, - ) - else: - for alias in elements[2:]: - if useflag_re.match(alias) is None: - writemsg( - _( - "--- Invalid USE flag alias for '%s' real USE flag for '%s' in '%s': '%s'\n" - ) - % (real_flag, atom, file_name, alias), - noiselevel=-1, - ) - else: - # Duplicated USE flag aliases in entries for different atoms - # matching the same package version are detected in getUseAliases(). - if any( - alias in v - for k, v in file_dict.get(atom.cp, {}) - .get(atom, {}) - .items() - if k != real_flag - ): - writemsg( - _( - "--- Duplicated USE flag alias for '%s' in '%s': '%s'\n" - ) - % (atom, file_name, alias), - noiselevel=-1, - ) - else: - file_dict.setdefault(atom.cp, {}).setdefault( - atom, {} - ).setdefault(real_flag, []).append(alias) - ret[repo.name] = file_dict - return ret - def _isStable(self, pkg): if self._user_config: try: @@ -650,65 +533,6 @@ class UseManager: return frozenset(stack_lists(useforce, incremental=True)) - def getUseAliases(self, pkg): - if hasattr(pkg, "eapi") and not eapi_has_use_aliases(pkg.eapi): - return {} - - cp = getattr(pkg, "cp", None) - if cp is None: - slot = dep_getslot(pkg) - repo = dep_getrepo(pkg) - pkg = _pkg_str(remove_slot(pkg), slot=slot, repo=repo) - cp = pkg.cp - - usealiases = {} - - if hasattr(pkg, "repo") and pkg.repo != Package.UNKNOWN_REPO: - repos = [] - try: - repos.extend(repo.name for repo in self.repositories[pkg.repo].masters) - except KeyError: - pass - repos.append(pkg.repo) - for repo in repos: - usealiases_dict = self._repo_usealiases_dict.get(repo, {}) - for real_flag, aliases in usealiases_dict.items(): - for alias in aliases: - if any( - alias in v for k, v in usealiases.items() if k != real_flag - ): - writemsg( - _("--- Duplicated USE flag alias for '%s%s%s': '%s'\n") - % (pkg.cpv, _repo_separator, pkg.repo, alias), - noiselevel=-1, - ) - else: - usealiases.setdefault(real_flag, []).append(alias) - cp_usealiases_dict = self._repo_pusealiases_dict.get(repo, {}).get(cp) - if cp_usealiases_dict: - usealiases_dict_list = ordered_by_atom_specificity( - cp_usealiases_dict, pkg - ) - for usealiases_dict in usealiases_dict_list: - for real_flag, aliases in usealiases_dict.items(): - for alias in aliases: - if any( - alias in v - for k, v in usealiases.items() - if k != real_flag - ): - writemsg( - _( - "--- Duplicated USE flag alias for '%s%s%s': '%s'\n" - ) - % (pkg.cpv, _repo_separator, pkg.repo, alias), - noiselevel=-1, - ) - else: - usealiases.setdefault(real_flag, []).append(alias) - - return usealiases - def getPUSE(self, pkg): cp = getattr(pkg, "cp", None) if cp is None: diff --git a/lib/portage/package/ebuild/_config/VirtualsManager.py b/lib/portage/package/ebuild/_config/VirtualsManager.py index a5473a94f..b0acf3c7d 100644 --- a/lib/portage/package/ebuild/_config/VirtualsManager.py +++ b/lib/portage/package/ebuild/_config/VirtualsManager.py @@ -107,11 +107,11 @@ class VirtualsManager: memo[id(self)] = result # immutable attributes (internal policy ensures lack of mutation) - # _treeVirtuals is initilised by _populate_treeVirtuals(). + # _treeVirtuals is initialised by _populate_treeVirtuals(). # Before that it's 'None'. result._treeVirtuals = self._treeVirtuals memo[id(self._treeVirtuals)] = self._treeVirtuals - # _dirVirtuals is initilised by __init__. + # _dirVirtuals is initialised by __init__. result._dirVirtuals = self._dirVirtuals memo[id(self._dirVirtuals)] = self._dirVirtuals diff --git a/lib/portage/package/ebuild/_config/env_var_validation.py b/lib/portage/package/ebuild/_config/env_var_validation.py index 82fddca03..e00070926 100644 --- a/lib/portage/package/ebuild/_config/env_var_validation.py +++ b/lib/portage/package/ebuild/_config/env_var_validation.py @@ -8,7 +8,7 @@ from portage.util import shlex_split def validate_cmd_var(v): """ - Validate an evironment variable value to see if it + Validate an environment variable value to see if it contains an executable command as the first token. returns (valid, token_list) where 'valid' is boolean and 'token_list' is the (possibly empty) list of tokens split by shlex. diff --git a/lib/portage/package/ebuild/_config/helper.py b/lib/portage/package/ebuild/_config/helper.py index 275d32cfe..1081b4da3 100644 --- a/lib/portage/package/ebuild/_config/helper.py +++ b/lib/portage/package/ebuild/_config/helper.py @@ -50,7 +50,7 @@ def ordered_by_atom_specificity(cpdict, pkg, repo=None): def prune_incremental(split): """ Prune off any parts of an incremental variable that are - made irrelevant by the latest occuring * or -*. This + made irrelevant by the latest occurring * or -*. This could be more aggressive but that might be confusing and the point is just to reduce noise a bit. """ diff --git a/lib/portage/package/ebuild/_config/meson.build b/lib/portage/package/ebuild/_config/meson.build new file mode 100644 index 000000000..e12d82e50 --- /dev/null +++ b/lib/portage/package/ebuild/_config/meson.build @@ -0,0 +1,17 @@ +py.install_sources( + [ + 'KeywordsManager.py', + 'LicenseManager.py', + 'LocationsManager.py', + 'MaskManager.py', + 'UseManager.py', + 'VirtualsManager.py', + 'env_var_validation.py', + 'features_set.py', + 'helper.py', + 'special_env_vars.py', + '__init__.py', + ], + subdir : 'portage/package/ebuild/_config', + pure : not native_extensions +) diff --git a/lib/portage/package/ebuild/_config/special_env_vars.py b/lib/portage/package/ebuild/_config/special_env_vars.py index 06ae3aa39..1a66192c9 100644 --- a/lib/portage/package/ebuild/_config/special_env_vars.py +++ b/lib/portage/package/ebuild/_config/special_env_vars.py @@ -72,8 +72,6 @@ env_blacklist = frozenset( ) ) -environ_whitelist = [] - # Whitelisted variables are always allowed to enter the ebuild # environment. Generally, this only includes special portage # variables. Ebuilds can unset variables that are not whitelisted @@ -82,274 +80,263 @@ environ_whitelist = [] # important to set our special BASH_ENV variable in the ebuild # environment in order to prevent sandbox from sourcing /etc/profile # in it's bashrc (causing major leakage). -environ_whitelist += [ - "ACCEPT_LICENSE", - "BASH_ENV", - "BASH_FUNC____in_portage_iuse%%", - "BROOT", - "BUILD_PREFIX", - "COLUMNS", - "D", - "DISTDIR", - "DOC_SYMLINKS_DIR", - "EAPI", - "EBUILD", - "EBUILD_FORCE_TEST", - "EBUILD_PHASE", - "EBUILD_PHASE_FUNC", - "ECLASSDIR", - "ECLASS_DEPTH", - "ED", - "EMERGE_FROM", - "ENV_UNSET", - "EPREFIX", - "EROOT", - "ESYSROOT", - "FEATURES", - "FILESDIR", - "HOME", - "MERGE_TYPE", - "NOCOLOR", - "PATH", - "PKGDIR", - "PKGUSE", - "PKG_LOGDIR", - "PKG_TMPDIR", - "PORTAGE_ACTUAL_DISTDIR", - "PORTAGE_ARCHLIST", - "PORTAGE_BASHRC_FILES", - "PORTAGE_BASHRC", - "PM_EBUILD_HOOK_DIR", - "PORTAGE_BINPKG_FILE", - "PORTAGE_BINPKG_TAR_OPTS", - "PORTAGE_BINPKG_TMPFILE", - "PORTAGE_BIN_PATH", - "PORTAGE_BUILDDIR", - "PORTAGE_BUILD_GROUP", - "PORTAGE_BUILD_USER", - "PORTAGE_BUNZIP2_COMMAND", - "PORTAGE_BZIP2_COMMAND", - "PORTAGE_COLORMAP", - "PORTAGE_COMPRESS", - "PORTAGE_COMPRESSION_COMMAND", - "PORTAGE_COMPRESS_EXCLUDE_SUFFIXES", - "PORTAGE_CONFIGROOT", - "PORTAGE_DEBUG", - "PORTAGE_DEPCACHEDIR", - "PORTAGE_DOHTML_UNWARNED_SKIPPED_EXTENSIONS", - "PORTAGE_DOHTML_UNWARNED_SKIPPED_FILES", - "PORTAGE_DOHTML_WARN_ON_SKIPPED_FILES", - "PORTAGE_EBUILD_EXIT_FILE", - "PORTAGE_FEATURES", - "PORTAGE_GID", - "PORTAGE_GRPNAME", - "PORTAGE_INTERNAL_CALLER", - "PORTAGE_INST_GID", - "PORTAGE_INST_UID", - "PORTAGE_IPC_DAEMON", - "PORTAGE_IUSE", - "PORTAGE_ECLASS_LOCATIONS", - "PORTAGE_LOG_FILE", - "PORTAGE_OVERRIDE_EPREFIX", - "PORTAGE_PIPE_FD", - "PORTAGE_PROPERTIES", - "PORTAGE_PYM_PATH", - "PORTAGE_PYTHON", - "PORTAGE_PYTHONPATH", - "PORTAGE_QUIET", - "PORTAGE_REPO_NAME", - "PORTAGE_REPOSITORIES", - "PORTAGE_RESTRICT", - "PORTAGE_SIGPIPE_STATUS", - "PORTAGE_SOCKS5_PROXY", - "PORTAGE_TMPDIR", - "PORTAGE_UPDATE_ENV", - "PORTAGE_USERNAME", - "PORTAGE_VERBOSE", - "PORTAGE_WORKDIR_MODE", - "PORTAGE_XATTR_EXCLUDE", - "PORTDIR", - "PORTDIR_OVERLAY", - "PREROOTPATH", - "PYTHONDONTWRITEBYTECODE", - "REPLACING_VERSIONS", - "REPLACED_BY_VERSION", - "ROOT", - "ROOTPATH", - "SANDBOX_LOG", - "SYSROOT", - "T", - "TMP", - "TMPDIR", - "USE_EXPAND", - "USE_ORDER", - "WORKDIR", - "XARGS", - "__PORTAGE_TEST_HARDLINK_LOCKS", -] - -# user config variables -environ_whitelist += ["DOC_SYMLINKS_DIR", "INSTALL_MASK", "PKG_INSTALL_MASK"] - -environ_whitelist += ["A", "AA", "CATEGORY", "P", "PF", "PN", "PR", "PV", "PVR"] - -# misc variables inherited from the calling environment -environ_whitelist += [ - "COLORTERM", - "DISPLAY", - "EDITOR", - "LESS", - "LESSOPEN", - "LOGNAME", - "LS_COLORS", - "PAGER", - "TERM", - "TERMCAP", - "USER", - "ftp_proxy", - "http_proxy", - "no_proxy", -] - -# tempdir settings -environ_whitelist += [ - "TMPDIR", - "TEMP", - "TMP", -] - -# localization settings -environ_whitelist += [ - "LANG", - "LC_COLLATE", - "LC_CTYPE", - "LC_MESSAGES", - "LC_MONETARY", - "LC_NUMERIC", - "LC_TIME", - "LC_PAPER", - "LC_ALL", -] - -# other variables inherited from the calling environment -environ_whitelist += [ - "CVS_RSH", - "ECHANGELOG_USER", - "GPG_AGENT_INFO", - "SSH_AGENT_PID", - "SSH_AUTH_SOCK", - "STY", - "WINDOW", - "XAUTHORITY", -] - -environ_whitelist = frozenset(environ_whitelist) +environ_whitelist = frozenset( + ( + "A", + "AA", + "ACCEPT_LICENSE", + "BASH_ENV", + "BASH_FUNC____in_portage_iuse%%", + "BINPKG_FORMAT", + "BROOT", + "BUILD_ID", + "BUILD_PREFIX", + "CATEGORY", + "COLUMNS", + "D", + "DISTDIR", + "DOC_SYMLINKS_DIR", + "EAPI", + "EBUILD", + "EBUILD_FORCE_TEST", + "EBUILD_PHASE", + "EBUILD_PHASE_FUNC", + "ECLASSDIR", + "ECLASS_DEPTH", + "ED", + "EMERGE_FROM", + "ENV_UNSET", + "EPREFIX", + "EROOT", + "ESYSROOT", + "FEATURES", + "FILESDIR", + "HOME", + "MERGE_TYPE", + "NOCOLOR", + "NO_COLOR", + "P", + "PATH", + "PF", + "PKGDIR", + "PKGUSE", + "PKG_LOGDIR", + "PKG_TMPDIR", + "PM_EBUILD_HOOK_DIR", + "PN", + "PORTAGE_ACTUAL_DISTDIR", + "PORTAGE_ARCHLIST", + "PORTAGE_BASHRC_FILES", + "PORTAGE_BASHRC", + "PORTAGE_BINPKG_FILE", + "PORTAGE_BINPKG_TAR_OPTS", + "PORTAGE_BINPKG_TMPFILE", + "PORTAGE_BIN_PATH", + "PORTAGE_BUILDDIR", + "PORTAGE_BUILD_GROUP", + "PORTAGE_BUILD_USER", + "PORTAGE_BUNZIP2_COMMAND", + "PORTAGE_BZIP2_COMMAND", + "PORTAGE_COLORMAP", + "PORTAGE_COMPRESS", + "PORTAGE_COMPRESSION_COMMAND", + "PORTAGE_COMPRESS_EXCLUDE_SUFFIXES", + "PORTAGE_CONFIGROOT", + "PORTAGE_DEBUG", + "PORTAGE_DEPCACHEDIR", + "PORTAGE_DOHTML_UNWARNED_SKIPPED_EXTENSIONS", + "PORTAGE_DOHTML_UNWARNED_SKIPPED_FILES", + "PORTAGE_DOHTML_WARN_ON_SKIPPED_FILES", + "PORTAGE_EBUILD_EXIT_FILE", + "PORTAGE_FEATURES", + "PORTAGE_GID", + "PORTAGE_GRPNAME", + "PORTAGE_INTERNAL_CALLER", + "PORTAGE_INST_GID", + "PORTAGE_INST_UID", + "PORTAGE_IPC_DAEMON", + "PORTAGE_IUSE", + "PORTAGE_ECLASS_LOCATIONS", + "PORTAGE_LOG_FILE", + "PORTAGE_OVERRIDE_EPREFIX", + "PORTAGE_PIPE_FD", + "PORTAGE_PROPERTIES", + "PORTAGE_PYM_PATH", + "PORTAGE_PYTHON", + "PORTAGE_PYTHONPATH", + "PORTAGE_QUIET", + "PORTAGE_REPO_REVISIONS", + "PORTAGE_REPO_NAME", + "PORTAGE_REPOSITORIES", + "PORTAGE_RESTRICT", + "PORTAGE_SIGPIPE_STATUS", + "PORTAGE_SOCKS5_PROXY", + "PORTAGE_TMPDIR", + "PORTAGE_UPDATE_ENV", + "PORTAGE_USERNAME", + "PORTAGE_VERBOSE", + "PORTAGE_WORKDIR_MODE", + "PORTAGE_XATTR_EXCLUDE", + "PORTDIR", + "PORTDIR_OVERLAY", + "PR", + "PREROOTPATH", + "PV", + "PVR", + "PYTHONDONTWRITEBYTECODE", + "REPLACING_VERSIONS", + "REPLACED_BY_VERSION", + "ROOT", + "ROOTPATH", + "SANDBOX_LOG", + "SYSROOT", + "T", + "TMP", + "TMPDIR", + "USE_EXPAND", + "USE_ORDER", + "WORKDIR", + "XARGS", + "__PORTAGE_TEST_HARDLINK_LOCKS", + # user config variables + "DOC_SYMLINKS_DIR", + "INSTALL_MASK", + "PKG_INSTALL_MASK", + # misc variables inherited from the calling environment + "COLORTERM", + "DISPLAY", + "EDITOR", + "LESS", + "LESSOPEN", + "LOGNAME", + "LS_COLORS", + "PAGER", + "TERM", + "TERMCAP", + "USER", + "ftp_proxy", + "http_proxy", + "https_proxy", + "no_proxy", + # tempdir settings + "TMPDIR", + "TEMP", + "TMP", + # localization settings + "LANG", + "LC_COLLATE", + "LC_CTYPE", + "LC_MESSAGES", + "LC_MONETARY", + "LC_NUMERIC", + "LC_TIME", + "LC_PAPER", + "LC_ALL", + # other variables inherited from the calling environment + "CVS_RSH", + "ECHANGELOG_USER", + "GPG_AGENT_INFO", + "SSH_AGENT_PID", + "SSH_AUTH_SOCK", + "STY", + "WINDOW", + "XAUTHORITY", + ) +) environ_whitelist_re = re.compile(r"^(CCACHE_|DISTCC_).*") # Filter selected variables in the config.environ() method so that # they don't needlessly propagate down into the ebuild environment. -environ_filter = [] - # Exclude anything that could be extremely long here (like SRC_URI) # since that could cause execve() calls to fail with E2BIG errors. For # example, see bug #262647. -environ_filter += [ - "DEPEND", - "RDEPEND", - "PDEPEND", - "SRC_URI", - "BDEPEND", - "IDEPEND", -] - -# misc variables inherited from the calling environment -environ_filter += [ - "INFOPATH", - "MANPATH", - "USER", -] - -# variables that break bash -environ_filter += [ - "HISTFILE", - "POSIXLY_CORRECT", -] - -# portage config variables and variables set directly by portage -environ_filter += [ - "ACCEPT_CHOSTS", - "ACCEPT_KEYWORDS", - "ACCEPT_PROPERTIES", - "ACCEPT_RESTRICT", - "AUTOCLEAN", - "BINPKG_COMPRESS", - "BINPKG_COMPRESS_FLAGS", - "CLEAN_DELAY", - "COLLISION_IGNORE", - "CONFIG_PROTECT", - "CONFIG_PROTECT_MASK", - "EGENCACHE_DEFAULT_OPTS", - "EMERGE_DEFAULT_OPTS", - "EMERGE_LOG_DIR", - "EMERGE_WARNING_DELAY", - "FETCHCOMMAND", - "FETCHCOMMAND_FTP", - "FETCHCOMMAND_HTTP", - "FETCHCOMMAND_HTTPS", - "FETCHCOMMAND_RSYNC", - "FETCHCOMMAND_SFTP", - "GENTOO_MIRRORS", - "NOCONFMEM", - "O", - "PORTAGE_BACKGROUND", - "PORTAGE_BACKGROUND_UNMERGE", - "PORTAGE_BINHOST", - "PORTAGE_BINPKG_FORMAT", - "PORTAGE_BUILDDIR_LOCKED", - "PORTAGE_CHECKSUM_FILTER", - "PORTAGE_ELOG_CLASSES", - "PORTAGE_ELOG_MAILFROM", - "PORTAGE_ELOG_MAILSUBJECT", - "PORTAGE_ELOG_MAILURI", - "PORTAGE_ELOG_SYSTEM", - "PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", - "PORTAGE_FETCH_RESUME_MIN_SIZE", - "PORTAGE_GPG_DIR", - "PORTAGE_GPG_KEY", - "PORTAGE_GPG_SIGNING_COMMAND", - "PORTAGE_IONICE_COMMAND", - "PORTAGE_PACKAGE_EMPTY_ABORT", - "PORTAGE_REPO_DUPLICATE_WARN", - "PORTAGE_RO_DISTDIRS", - "PORTAGE_RSYNC_EXTRA_OPTS", - "PORTAGE_RSYNC_OPTS", - "PORTAGE_RSYNC_RETRIES", - "PORTAGE_SSH_OPTS", - "PORTAGE_SYNC_STALE", - "PORTAGE_USE", - "PORTAGE_LOG_FILTER_FILE_CMD", - "PORTAGE_LOGDIR", - "PORTAGE_LOGDIR_CLEAN", - "QUICKPKG_DEFAULT_OPTS", - "REPOMAN_DEFAULT_OPTS", - "RESUMECOMMAND", - "RESUMECOMMAND_FTP", - "RESUMECOMMAND_HTTP", - "RESUMECOMMAND_HTTPS", - "RESUMECOMMAND_RSYNC", - "RESUMECOMMAND_SFTP", - "SIGNED_OFF_BY", - "UNINSTALL_IGNORE", - "USE_EXPAND_HIDDEN", - "USE_ORDER", - "__PORTAGE_HELPER", -] - -# No longer supported variables -environ_filter += ["SYNC"] - -environ_filter = frozenset(environ_filter) +environ_filter = frozenset( + ( + "DEPEND", + "RDEPEND", + "PDEPEND", + "SRC_URI", + "BDEPEND", + "IDEPEND", + # misc variables inherited from the calling environment + "INFOPATH", + "MANPATH", + "USER", + # variables that break bash + "HISTFILE", + "POSIXLY_CORRECT", + # portage config variables and variables set directly by portage + "ACCEPT_CHOSTS", + "ACCEPT_KEYWORDS", + "ACCEPT_PROPERTIES", + "ACCEPT_RESTRICT", + "AUTOCLEAN", + "BINPKG_COMPRESS", + "BINPKG_COMPRESS_FLAGS", + "CLEAN_DELAY", + "COLLISION_IGNORE", + "CONFIG_PROTECT", + "CONFIG_PROTECT_MASK", + "EGENCACHE_DEFAULT_OPTS", + "EMERGE_DEFAULT_OPTS", + "EMERGE_LOG_DIR", + "EMERGE_WARNING_DELAY", + "FETCHCOMMAND", + "FETCHCOMMAND_FTP", + "FETCHCOMMAND_HTTP", + "FETCHCOMMAND_HTTPS", + "FETCHCOMMAND_RSYNC", + "FETCHCOMMAND_SFTP", + "GENTOO_MIRRORS", + "NOCONFMEM", + "O", + "PORTAGE_BACKGROUND", + "PORTAGE_BACKGROUND_UNMERGE", + "PORTAGE_BINHOST", + "PORTAGE_BINPKG_FORMAT", + "PORTAGE_BUILDDIR_LOCKED", + "PORTAGE_CHECKSUM_FILTER", + "PORTAGE_ELOG_CLASSES", + "PORTAGE_ELOG_MAILFROM", + "PORTAGE_ELOG_MAILSUBJECT", + "PORTAGE_ELOG_MAILURI", + "PORTAGE_ELOG_SYSTEM", + "PORTAGE_FETCH_CHECKSUM_TRY_MIRRORS", + "PORTAGE_FETCH_RESUME_MIN_SIZE", + "PORTAGE_GPG_DIR", + "PORTAGE_GPG_KEY", + "PORTAGE_GPG_SIGNING_COMMAND", + "PORTAGE_IONICE_COMMAND", + "PORTAGE_PACKAGE_EMPTY_ABORT", + "PORTAGE_REPO_DUPLICATE_WARN", + "PORTAGE_RO_DISTDIRS", + "PORTAGE_RSYNC_EXTRA_OPTS", + "PORTAGE_RSYNC_OPTS", + "PORTAGE_RSYNC_RETRIES", + "PORTAGE_SSH_OPTS", + "PORTAGE_SYNC_STALE", + "PORTAGE_TRUST_HELPER", + "PORTAGE_USE", + "PORTAGE_LOG_FILTER_FILE_CMD", + "PORTAGE_LOGDIR", + "PORTAGE_LOGDIR_CLEAN", + "QUICKPKG_DEFAULT_OPTS", + "REPOMAN_DEFAULT_OPTS", + "RESUMECOMMAND", + "RESUMECOMMAND_FTP", + "RESUMECOMMAND_HTTP", + "RESUMECOMMAND_HTTPS", + "RESUMECOMMAND_RSYNC", + "RESUMECOMMAND_SFTP", + "UNINSTALL_IGNORE", + "USE_EXPAND_HIDDEN", + "USE_ORDER", + "__PORTAGE_HELPER", + # No longer supported variables + "SYNC", + ) +) # Variables that are not allowed to have per-repo or per-package # settings. @@ -375,4 +362,5 @@ validate_commands = ( case_insensitive_vars = ( "AUTOCLEAN", "NOCOLOR", + "NO_COLOR", ) diff --git a/lib/portage/package/ebuild/_config/unpack_dependencies.py b/lib/portage/package/ebuild/_config/unpack_dependencies.py deleted file mode 100644 index 58b2fb16e..000000000 --- a/lib/portage/package/ebuild/_config/unpack_dependencies.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage import os, _supported_eapis -from portage.dep import use_reduce -from portage.eapi import eapi_has_automatic_unpack_dependencies -from portage.exception import InvalidDependString -from portage.localization import _ -from portage.util import grabfile, writemsg - - -def load_unpack_dependencies_configuration(repositories): - repo_dict = {} - for repo in repositories.repos_with_profiles(): - for eapi in _supported_eapis: - if eapi_has_automatic_unpack_dependencies(eapi): - file_name = os.path.join( - repo.location, "profiles", "unpack_dependencies", eapi - ) - lines = grabfile(file_name, recursive=True) - for line in lines: - elements = line.split() - suffix = elements[0].lower() - if len(elements) == 1: - writemsg( - _( - "--- Missing unpack dependencies for '%s' suffix in '%s'\n" - ) - % (suffix, file_name) - ) - depend = " ".join(elements[1:]) - try: - use_reduce(depend, eapi=eapi) - except InvalidDependString as e: - writemsg( - _( - "--- Invalid unpack dependencies for '%s' suffix in '%s': '%s'\n" - % (suffix, file_name, e) - ) - ) - else: - repo_dict.setdefault(repo.name, {}).setdefault(eapi, {})[ - suffix - ] = depend - - ret = {} - for repo in repositories.repos_with_profiles(): - for repo_name in [x.name for x in repo.masters] + [repo.name]: - for eapi in repo_dict.get(repo_name, {}): - for suffix, depend in ( - repo_dict.get(repo_name, {}).get(eapi, {}).items() - ): - ret.setdefault(repo.name, {}).setdefault(eapi, {})[suffix] = depend - - return ret diff --git a/lib/portage/package/ebuild/_ipc/ExitCommand.py b/lib/portage/package/ebuild/_ipc/ExitCommand.py index be419e9b7..b7c970353 100644 --- a/lib/portage/package/ebuild/_ipc/ExitCommand.py +++ b/lib/portage/package/ebuild/_ipc/ExitCommand.py @@ -5,7 +5,6 @@ from portage.package.ebuild._ipc.IpcCommand import IpcCommand class ExitCommand(IpcCommand): - __slots__ = ( "exitcode", "reply_hook", @@ -17,7 +16,6 @@ class ExitCommand(IpcCommand): self.exitcode = None def __call__(self, argv): - if self.exitcode is not None: # Ignore all but the first call, since if die is called # then we certainly want to honor that exitcode, even diff --git a/lib/portage/package/ebuild/_ipc/IpcCommand.py b/lib/portage/package/ebuild/_ipc/IpcCommand.py index 2c4b9d8be..763cdb30b 100644 --- a/lib/portage/package/ebuild/_ipc/IpcCommand.py +++ b/lib/portage/package/ebuild/_ipc/IpcCommand.py @@ -3,7 +3,6 @@ class IpcCommand: - __slots__ = () def __call__(self, argv): diff --git a/lib/portage/package/ebuild/_ipc/QueryCommand.py b/lib/portage/package/ebuild/_ipc/QueryCommand.py index f8f464516..faf1baa0a 100644 --- a/lib/portage/package/ebuild/_ipc/QueryCommand.py +++ b/lib/portage/package/ebuild/_ipc/QueryCommand.py @@ -10,12 +10,11 @@ from portage.eapi import eapi_has_repo_deps from portage.elog import messages as elog_messages from portage.exception import InvalidAtom from portage.package.ebuild._ipc.IpcCommand import IpcCommand -from portage.util import normalize_path +from portage.util import normalize_path, no_color from portage.versions import best class QueryCommand(IpcCommand): - __slots__ = ( "phase", "settings", @@ -53,7 +52,7 @@ class QueryCommand(IpcCommand): root = normalize_path(root or os.sep).rstrip(os.sep) + os.sep if root not in db: - return ("", "%s: Invalid ROOT: %s\n" % (cmd, root), 3) + return ("", f"{cmd}: Invalid ROOT: {root}\n", 3) portdb = db[root]["porttree"].dbapi vardb = db[root]["vartree"].dbapi @@ -63,12 +62,12 @@ class QueryCommand(IpcCommand): try: atom = Atom(args[0], allow_repo=allow_repo) except InvalidAtom: - return ("", "%s: Invalid atom: %s\n" % (cmd, args[0]), 2) + return ("", f"{cmd}: Invalid atom: {args[0]}\n", 2) try: atom = Atom(args[0], allow_repo=allow_repo, eapi=eapi) except InvalidAtom as e: - warnings.append("QA Notice: %s: %s" % (cmd, e)) + warnings.append(f"QA Notice: {cmd}: {e}") use = self.settings.get("PORTAGE_BUILT_USE") if use is None: @@ -88,7 +87,7 @@ class QueryCommand(IpcCommand): return ("", warnings_str, returncode) if cmd == "best_version": m = best(vardb.match(atom)) - return ("%s\n" % m, warnings_str, 0) + return (f"{m}\n", warnings_str, 0) if cmd in ( "master_repositories", "repository_path", @@ -98,7 +97,7 @@ class QueryCommand(IpcCommand): ): repo = _repo_name_re.match(args[0]) if repo is None: - return ("", "%s: Invalid repository: %s\n" % (cmd, args[0]), 2) + return ("", f"{cmd}: Invalid repository: {args[0]}\n", 2) try: repo = portdb.repositories[args[0]] except KeyError: @@ -106,15 +105,15 @@ class QueryCommand(IpcCommand): if cmd == "master_repositories": return ( - "%s\n" % " ".join(x.name for x in repo.masters), + f"{' '.join(x.name for x in repo.masters)}\n", warnings_str, 0, ) if cmd == "repository_path": - return ("%s\n" % repo.location, warnings_str, 0) + return (f"{repo.location}\n", warnings_str, 0) if cmd == "available_eclasses": return ( - "%s\n" % " ".join(sorted(repo.eclass_db.eclasses)), + f"{' '.join(sorted(repo.eclass_db.eclasses))}\n", warnings_str, 0, ) @@ -123,7 +122,7 @@ class QueryCommand(IpcCommand): eclass = repo.eclass_db.eclasses[args[1]] except KeyError: return ("", warnings_str, 1) - return ("%s\n" % eclass.location, warnings_str, 0) + return (f"{eclass.location}\n", warnings_str, 0) if cmd == "license_path": paths = reversed( [ @@ -133,9 +132,9 @@ class QueryCommand(IpcCommand): ) for path in paths: if os.path.exists(path): - return ("%s\n" % path, warnings_str, 0) + return (f"{path}\n", warnings_str, 0) return ("", warnings_str, 1) - return ("", "Invalid command: %s\n" % cmd, 3) + return ("", f"Invalid command: {cmd}\n", 3) def _elog(self, elog_funcname, lines): """ @@ -150,9 +149,7 @@ class QueryCommand(IpcCommand): elog_func = getattr(elog_messages, elog_funcname) global_havecolor = portage.output.havecolor try: - portage.output.havecolor = self.settings.get( - "NOCOLOR", "false" - ).lower() in ("no", "false") + portage.output.havecolor = not no_color(self.settings) for line in lines: elog_func(line, phase=phase, key=self.settings.mycpv, out=out) finally: diff --git a/lib/portage/package/ebuild/_ipc/meson.build b/lib/portage/package/ebuild/_ipc/meson.build new file mode 100644 index 000000000..18c0ac1cf --- /dev/null +++ b/lib/portage/package/ebuild/_ipc/meson.build @@ -0,0 +1,10 @@ +py.install_sources( + [ + 'ExitCommand.py', + 'IpcCommand.py', + 'QueryCommand.py', + '__init__.py', + ], + subdir : 'portage/package/ebuild/_ipc', + pure : not native_extensions +) diff --git a/lib/portage/package/ebuild/_metadata_invalid.py b/lib/portage/package/ebuild/_metadata_invalid.py index b42adbcf2..310b21e9e 100644 --- a/lib/portage/package/ebuild/_metadata_invalid.py +++ b/lib/portage/package/ebuild/_metadata_invalid.py @@ -10,7 +10,6 @@ from portage.elog.messages import eerror def eapi_invalid(self, cpv, repo_name, settings, eapi_var, eapi_parsed, eapi_lineno): - msg = [] msg.extend( textwrap.wrap( @@ -25,9 +24,9 @@ def eapi_invalid(self, cpv, repo_name, settings, eapi_var, eapi_parsed, eapi_lin if not eapi_parsed: # None means the assignment was not found, while an - # empty string indicates an (invalid) empty assingment. + # empty string indicates an (invalid) empty assignment. msg.append( - "\tvalid EAPI assignment must" " occur on or before line: %s" % eapi_lineno + f"\tvalid EAPI assignment must occur on or before line: {eapi_lineno}" ) else: msg.append( diff --git a/lib/portage/package/ebuild/_parallel_manifest/ManifestProcess.py b/lib/portage/package/ebuild/_parallel_manifest/ManifestProcess.py index 7bf5dd141..ec2d5bdfc 100644 --- a/lib/portage/package/ebuild/_parallel_manifest/ManifestProcess.py +++ b/lib/portage/package/ebuild/_parallel_manifest/ManifestProcess.py @@ -1,6 +1,8 @@ -# Copyright 2012 Gentoo Foundation +# Copyright 2012-2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 +import functools + import portage from portage import os from portage.exception import FileNotFound, PermissionDenied, PortagePackageException @@ -9,16 +11,29 @@ from portage.util._async.ForkProcess import ForkProcess class ManifestProcess(ForkProcess): - __slots__ = ("cp", "distdir", "fetchlist_dict", "repo_config") MODIFIED = 16 - def _run(self): - mf = self.repo_config.load_manifest( - os.path.join(self.repo_config.location, self.cp), + def _start(self): + self.target = functools.partial( + self._target, + self.cp, self.distdir, - fetchlist_dict=self.fetchlist_dict, + self.fetchlist_dict, + self.repo_config, + ) + super()._start() + + @staticmethod + def _target(cp, distdir, fetchlist_dict, repo_config): + """ + TODO: Make all arguments picklable for the multiprocessing spawn start method. + """ + mf = repo_config.load_manifest( + os.path.join(repo_config.location, cp), + distdir, + fetchlist_dict=fetchlist_dict, ) try: @@ -31,22 +46,18 @@ class ManifestProcess(ForkProcess): return 1 except PortagePackageException as e: - portage.writemsg(("!!! %s\n") % (e,), noiselevel=-1) + portage.writemsg(f"!!! {e}\n", noiselevel=-1) return 1 try: modified = mf.write(sign=False) except PermissionDenied as e: portage.writemsg( - "!!! %s: %s\n" - % ( - _("Permission Denied"), - e, - ), + f"!!! {_('Permission Denied')}: {e}\n", noiselevel=-1, ) return 1 else: if modified: - return self.MODIFIED + return ManifestProcess.MODIFIED return os.EX_OK diff --git a/lib/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py b/lib/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py index 4599e2d50..da7529277 100644 --- a/lib/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py +++ b/lib/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py @@ -18,9 +18,8 @@ class ManifestScheduler(AsyncScheduler): gpg_cmd=None, gpg_vars=None, force_sign_key=None, - **kwargs + **kwargs, ): - AsyncScheduler.__init__(self, **kwargs) self._portdb = portdb @@ -41,8 +40,7 @@ class ManifestScheduler(AsyncScheduler): # and in order to reduce latency in case of a signal interrupt. cp_all = self._portdb.cp_all for category in sorted(self._portdb.categories): - for cp in cp_all(categories=(category,)): - yield cp + yield from cp_all(categories=(category,)) def _iter_tasks(self): portdb = self._portdb @@ -94,7 +92,6 @@ class ManifestScheduler(AsyncScheduler): ) def _task_exit(self, task): - if task.returncode != os.EX_OK: if not self._terminated_tasks: portage.writemsg( diff --git a/lib/portage/package/ebuild/_parallel_manifest/ManifestTask.py b/lib/portage/package/ebuild/_parallel_manifest/ManifestTask.py index df279dab6..87aa46de5 100644 --- a/lib/portage/package/ebuild/_parallel_manifest/ManifestTask.py +++ b/lib/portage/package/ebuild/_parallel_manifest/ManifestTask.py @@ -21,7 +21,6 @@ from .ManifestProcess import ManifestProcess class ManifestTask(CompositeTask): - __slots__ = ( "cp", "distdir", @@ -233,7 +232,7 @@ class ManifestTask(CompositeTask): "rb", ) as f: return self._PGP_HEADER not in f.readline() - except IOError as e: + except OSError as e: if e.errno in (errno.ENOENT, errno.ESTALE): return False raise diff --git a/lib/portage/package/ebuild/_parallel_manifest/meson.build b/lib/portage/package/ebuild/_parallel_manifest/meson.build new file mode 100644 index 000000000..14b3fa659 --- /dev/null +++ b/lib/portage/package/ebuild/_parallel_manifest/meson.build @@ -0,0 +1,10 @@ +py.install_sources( + [ + 'ManifestProcess.py', + 'ManifestScheduler.py', + 'ManifestTask.py', + '__init__.py', + ], + subdir : 'portage/package/ebuild/_parallel_manifest', + pure : not native_extensions +) diff --git a/lib/portage/package/ebuild/config.py b/lib/portage/package/ebuild/config.py index b4d6862a3..67fd1bb18 100644 --- a/lib/portage/package/ebuild/config.py +++ b/lib/portage/package/ebuild/config.py @@ -1,4 +1,4 @@ -# Copyright 2010-2021 Gentoo Authors +# Copyright 2010-2023 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 __all__ = [ @@ -29,7 +29,6 @@ portage.proxy.lazyimport.lazyimport( "portage.dbapi.vartree:vartree", "portage.package.ebuild.doebuild:_phase_func_map", "portage.util.compression_probe:_compressors", - "portage.util.locale:check_locale,split_LC_ALL", ) from portage import bsd_chflags, load_mod, os, selinux, _unicode_decode from portage.const import ( @@ -41,6 +40,7 @@ from portage.const import ( PORTAGE_BASE_PATH, PRIVATE_PATH, PROFILE_PATH, + SUPPORTED_GENTOO_BINPKG_FORMATS, USER_CONFIG_PATH, USER_VIRTUALS_FILE, ) @@ -104,9 +104,6 @@ from portage.package.ebuild._config.helper import ( ordered_by_atom_specificity, prune_incremental, ) -from portage.package.ebuild._config.unpack_dependencies import ( - load_unpack_dependencies_configuration, -) _feature_flags_cache = {} @@ -135,8 +132,7 @@ def autouse(myvartree, use_cache=1, mysettings=None): def check_config_instance(test): if not isinstance(test, config): raise TypeError( - "Invalid type for config object: %s (should be %s)" - % (test.__class__, config) + f"Invalid type for config object: {test.__class__} (should be {config})" ) @@ -148,7 +144,7 @@ def best_from_dict(key, top_dict, key_order, EmptyOnError=1, FullCopy=1, AllowEm return top_dict[x][key] if EmptyOnError: return "" - raise KeyError("Key not found in list; '%s'" % key) + raise KeyError(f"Key not found in list; '{key}'") def _lazy_iuse_regex(iuse_implicit): @@ -160,7 +156,7 @@ def _lazy_iuse_regex(iuse_implicit): # Escape anything except ".*" which is supposed to pass through from # _get_implicit_iuse(). regex = sorted(re.escape(x) for x in iuse_implicit) - regex = "^(%s)$" % "|".join(regex) + regex = f"^({'|'.join(regex)})$" regex = regex.replace("\\.\\*", ".*") return regex @@ -168,7 +164,7 @@ def _lazy_iuse_regex(iuse_implicit): class _iuse_implicit_match_cache: def __init__(self, settings): self._iuse_implicit_re = re.compile( - "^(%s)$" % "|".join(settings._get_implicit_iuse()) + f"^({'|'.join(settings._get_implicit_iuse())})$" ) self._cache = {} @@ -202,7 +198,6 @@ class config: _deprecated_keys = { "PORTAGE_LOGDIR": "PORT_LOGDIR", "PORTAGE_LOGDIR_CLEAN": "PORT_LOGDIR_CLEAN", - "SIGNED_OFF_BY": "DCO_SIGNED_OFF_BY", } _setcpv_aux_keys = ( @@ -334,7 +329,6 @@ class config: self.profiles = clone.profiles self.packages = clone.packages self.repositories = clone.repositories - self.unpack_dependencies = clone.unpack_dependencies self._default_features_use = clone._default_features_use self._iuse_effective = clone._iuse_effective self._iuse_implicit_match = clone._iuse_implicit_match @@ -428,7 +422,6 @@ class config: eprefix = locations_manager.eprefix config_root = locations_manager.config_root sysroot = locations_manager.sysroot - esysroot = locations_manager.esysroot broot = locations_manager.broot abs_user_config = locations_manager.abs_user_config make_conf_paths = [ @@ -471,6 +464,7 @@ class config: locations_manager.set_root_override(make_conf.get("ROOT")) target_root = locations_manager.target_root eroot = locations_manager.eroot + esysroot = locations_manager.esysroot self.global_config_path = locations_manager.global_config_path # The expand_map is used for variable substitution @@ -489,14 +483,7 @@ class config: # interaction with the calling environment that might # lead to unexpected results. - env_d = ( - getconfig( - os.path.join(eroot, "etc", "profile.env"), - tolerant=tolerant, - expand=False, - ) - or {} - ) + env_d = self._get_env_d(broot=broot, eroot=eroot, tolerant=tolerant) expand_map = env_d.copy() self._expand_map = expand_map @@ -563,9 +550,7 @@ class config: user_auxdbmodule is not None and user_auxdbmodule in self._module_aliases ): - warnings.warn( - "'%s' is deprecated: %s" % (user_auxdbmodule, modules_file) - ) + warnings.warn(f"'{user_auxdbmodule}' is deprecated: {modules_file}") self.modules["default"] = { "portdbapi.auxdbmodule": "portage.cache.flat_hash.mtime_md5_database", @@ -598,9 +583,9 @@ class config: env = os.environ # Avoid potential UnicodeDecodeError exceptions later. - env_unicode = dict( - (_unicode_decode(k), _unicode_decode(v)) for k, v in env.items() - ) + env_unicode = { + _unicode_decode(k): _unicode_decode(v) for k, v in env.items() + } self.backupenv = env_unicode @@ -716,7 +701,7 @@ class config: ) for x in profiles_complex ] - except EnvironmentError as e: + except OSError as e: _raise_exc(e) self.packages = tuple(stack_lists(packages_list, incremental=1)) @@ -729,10 +714,6 @@ class config: x = Atom(x.lstrip("*")) self.prevmaskdict.setdefault(x.cp, []).append(x) - self.unpack_dependencies = load_unpack_dependencies_configuration( - self.repositories - ) - mygcfg = {} if profiles_complex: mygcfg_dlists = [] @@ -884,10 +865,10 @@ class config: # Initialize all USE related variables we track ourselves. self.usemask = self._use_manager.getUseMask() self.useforce = self._use_manager.getUseForce() - self.configdict["conf"][ - "USE" - ] = self._use_manager.extract_global_USE_changes( - self.configdict["conf"].get("USE", "") + self.configdict["conf"]["USE"] = ( + self._use_manager.extract_global_USE_changes( + self.configdict["conf"].get("USE", "") + ) ) # Read license_groups and optionally license_groups and package.license from user config @@ -897,10 +878,10 @@ class config: user_config=local_config, ) # Extract '*/*' entries from package.license - self.configdict["conf"][ - "ACCEPT_LICENSE" - ] = self._license_manager.extract_global_changes( - self.configdict["conf"].get("ACCEPT_LICENSE", "") + self.configdict["conf"]["ACCEPT_LICENSE"] = ( + self._license_manager.extract_global_changes( + self.configdict["conf"].get("ACCEPT_LICENSE", "") + ) ) # profile.bashrc @@ -1086,9 +1067,9 @@ class config: # reasonable defaults; this is important as without USE_ORDER, # USE will always be "" (nothing set)! if "USE_ORDER" not in self: - self[ - "USE_ORDER" - ] = "env:pkg:conf:defaults:pkginternal:features:repo:env.d" + self["USE_ORDER"] = ( + "env:pkg:conf:defaults:pkginternal:features:repo:env.d" + ) self.backup_changes("USE_ORDER") if "CBUILD" not in self and "CHOST" in self: @@ -1120,7 +1101,6 @@ class config: except OSError: pass else: - if portage.data._unprivileged_mode(eroot_or_parent, eroot_st): unprivileged = True @@ -1217,6 +1197,61 @@ class config: if mycpv: self.setcpv(mycpv) + def _get_env_d(self, broot, eroot, tolerant): + broot_only_variables = ( + "PATH", + "PREROOTPATH", + "ROOTPATH", + ) + eroot_only_variables = ( + "CONFIG_PROTECT", + "CONFIG_PROTECT_MASK", + "INFODIR", + "INFOPATH", + "MANPATH", + "PKG_CONFIG_.*", + ) + + broot_only_variables_re = re.compile(r"^(%s)$" % "|".join(broot_only_variables)) + eroot_only_variables_re = re.compile(r"^(%s)$" % "|".join(eroot_only_variables)) + + broot_env_d_path = os.path.join(broot or "/", "etc", "profile.env") + eroot_env_d_path = os.path.join(eroot or "/", "etc", "profile.env") + + if ( + os.path.exists(broot_env_d_path) + and os.path.exists(eroot_env_d_path) + and os.path.samefile(broot_env_d_path, eroot_env_d_path) + ): + broot_env_d = ( + getconfig(broot_env_d_path, tolerant=tolerant, expand=False) or {} + ) + eroot_env_d = broot_env_d + else: + broot_env_d = ( + getconfig(broot_env_d_path, tolerant=tolerant, expand=False) or {} + ) + eroot_env_d = ( + getconfig(eroot_env_d_path, tolerant=tolerant, expand=False) or {} + ) + + env_d = {} + + for k in broot_env_d.keys() | eroot_env_d.keys(): + if broot_only_variables_re.match(k): + if k in broot_env_d: + env_d[k] = broot_env_d[k] + elif eroot_only_variables_re.match(k): + if k in eroot_env_d: + env_d[k] = eroot_env_d[k] + else: + if k in eroot_env_d: + env_d[k] = eroot_env_d[k] + elif k in broot_env_d: + env_d[k] = broot_env_d[k] + + return env_d + def _init_iuse(self): self._iuse_effective = self._calc_iuse_effective() self._iuse_implicit_match = _iuse_implicit_match_cache(self) @@ -1297,7 +1332,7 @@ class config: _("!!! Directory initialization failed: '%s'\n") % mydir, noiselevel=-1, ) - writemsg("!!! %s\n" % str(e), noiselevel=-1) + writemsg(f"!!! {str(e)}\n", noiselevel=-1) @property def _keywords_manager(self): @@ -1513,6 +1548,15 @@ class config: noiselevel=-1, ) + binpkg_format = self.get("BINPKG_FORMAT") + if binpkg_format: + if binpkg_format not in SUPPORTED_GENTOO_BINPKG_FORMATS: + writemsg( + "!!! BINPKG_FORMAT contains invalid or " + "unsupported format: %s" % binpkg_format, + noiselevel=-1, + ) + binpkg_compression = self.get("BINPKG_COMPRESS") if binpkg_compression: try: @@ -1520,10 +1564,21 @@ class config: except KeyError as e: writemsg( "!!! BINPKG_COMPRESS contains invalid or " - "unsupported compression method: %s" % e.args[0], + "unsupported compression method: %s\n" % e.args[0], noiselevel=-1, ) else: + if ( + self.get( + f"BINPKG_COMPRESS_FLAGS_{binpkg_compression.upper()}", None + ) + is not None + ): + compression["compress"] = compression["compress"].replace( + "${BINPKG_COMPRESS_FLAGS}", + f"${{BINPKG_COMPRESS_FLAGS_{binpkg_compression.upper()}}}", + ) + try: compression_binary = shlex_split( portage.util.varexpand(compression["compress"], mydict=self) @@ -1531,7 +1586,7 @@ class config: except IndexError as e: writemsg( "!!! BINPKG_COMPRESS contains invalid or " - "unsupported compression method: %s" % e.args[0], + "unsupported compression method: %s\n" % e.args[0], noiselevel=-1, ) else: @@ -1539,7 +1594,7 @@ class config: missing_package = compression["package"] writemsg( "!!! BINPKG_COMPRESS unsupported %s. " - "Missing package: %s" + "Missing package: %s\n" % (binpkg_compression, missing_package), noiselevel=-1, ) @@ -1614,7 +1669,6 @@ class config: self.regenerate() class _lazy_vars: - __slots__ = ("built_use", "settings", "values") def __init__(self, built_use, settings): @@ -1634,14 +1688,14 @@ class config: if use is None: use = frozenset(settings["PORTAGE_USE"].split()) - values[ - "ACCEPT_LICENSE" - ] = settings._license_manager.get_prunned_accept_license( - settings.mycpv, - use, - settings.get("LICENSE", ""), - settings.get("SLOT"), - settings.get("PORTAGE_REPO_NAME"), + values["ACCEPT_LICENSE"] = ( + settings._license_manager.get_prunned_accept_license( + settings.mycpv, + use, + settings.get("LICENSE", ""), + settings.get("SLOT"), + settings.get("PORTAGE_REPO_NAME"), + ) ) values["PORTAGE_PROPERTIES"] = self._flatten("PROPERTIES", use, settings) values["PORTAGE_RESTRICT"] = self._flatten("RESTRICT", use, settings) @@ -1684,14 +1738,14 @@ class config: def __getitem__(self, key): prefix = key.lower() + "_" prefix_len = len(prefix) - expand_flags = set( + expand_flags = { x[prefix_len:] for x in self._use if x[:prefix_len] == prefix - ) + } var_split = self._use_expand_dict.get(key, "").split() # Preserve the order of var_split because it can matter for things # like LINGUAS. var_split = [x for x in var_split if x in expand_flags] - var_split.extend(expand_flags.difference(var_split)) + var_split.extend(sorted(expand_flags.difference(var_split))) has_wildcard = "*" in expand_flags if has_wildcard: var_split = [x for x in var_split if x != "*"] @@ -2059,6 +2113,9 @@ class config: "test" in restrict and not "all" in allow_test and not ("test_network" in properties and "network" in allow_test) + and not ( + "test_privileged" in properties and "privileged" in allow_test + ) ) if restrict_test and "test" in self.features: @@ -2122,7 +2179,7 @@ class config: "fi; " "[[ -n ${___PORTAGE_IUSE_HASH[$1]} ]]; " "}" - ) % " ".join('["%s"]=1' % x for x in portage_iuse) + ) % " ".join(f'["{x}"]=1' for x in portage_iuse) else: portage_iuse = self._get_implicit_iuse() portage_iuse.update(explicit_iuse) @@ -2149,7 +2206,9 @@ class config: # "test" is in IUSE and USE=test is masked, so execution # of src_test() probably is not reliable. Therefore, # temporarily disable FEATURES=test just for this package. - self["FEATURES"] = " ".join(x for x in self.features if x != "test") + self["FEATURES"] = " ".join( + x for x in sorted(self.features) if x != "test" + ) # Allow _* flags from USE_EXPAND wildcards to pass through here. use.difference_update( @@ -2164,7 +2223,7 @@ class config: # Use the calculated USE flags to regenerate the USE_EXPAND flags so # that they are consistent. For optimal performance, use slice # comparison instead of startswith(). - use_expand_split = set(x.lower() for x in self.get("USE_EXPAND", "").split()) + use_expand_split = {x.lower() for x in self.get("USE_EXPAND", "").split()} lazy_use_expand = self._lazy_use_expand( self, unfiltered_use, @@ -2175,7 +2234,7 @@ class config: self._use_expand_dict, ) - use_expand_iuses = dict((k, set()) for k in use_expand_split) + use_expand_iuses = {k: set() for k in use_expand_split} for x in portage_iuse: x_split = x.split("_") if len(x_split) == 1: @@ -2240,7 +2299,7 @@ class config: if k in protected_keys or k in non_user_variables: writemsg( "!!! Illegal variable " - + "'%s' assigned in '%s'\n" % (k, penvfile), + + f"'{k}' assigned in '{penvfile}'\n", noiselevel=-1, ) elif k in incrementals: @@ -2639,14 +2698,13 @@ class config: def reload(self): """Reload things like /etc/profile.env that can change during runtime.""" - env_d_filename = os.path.join(self["EROOT"], "etc", "profile.env") self.configdict["env.d"].clear() - env_d = getconfig(env_d_filename, tolerant=self._tolerant, expand=False) - if env_d: - # env_d will be None if profile.env doesn't exist. - for k in self._env_d_blacklist: - env_d.pop(k, None) - self.configdict["env.d"].update(env_d) + env_d = self._get_env_d( + broot=self["BROOT"], eroot=self["EROOT"], tolerant=self._tolerant + ) + for k in self._env_d_blacklist: + env_d.pop(k, None) + self.configdict["env.d"].update(env_d) def regenerate(self, useonly=0, use_cache=None): """ @@ -2739,10 +2797,8 @@ class config: myflags = set() for mykey, incremental_list in increment_lists.items(): - myflags.clear() for mysplit in incremental_list: - for x in mysplit: if x == "-*": # "-*" is a special "minus" var that means "unset all settings". @@ -2787,7 +2843,7 @@ class config: use_expand_unprefixed = self.get("USE_EXPAND_UNPREFIXED", "").split() - # In order to best accomodate the long-standing practice of + # In order to best accommodate the long-standing practice of # setting default USE_EXPAND variables in the profile's # make.defaults, we translate these variables into their # equivalent USE flags so that useful incremental behavior @@ -2842,7 +2898,6 @@ class config: iuse = [x.lstrip("+-") for x in iuse.split()] myflags = set() for curdb in self.uvlist: - for k in use_expand_unprefixed: v = curdb.get(k) if v is None: @@ -2991,9 +3046,9 @@ class config: for k in use_expand: prefix = k.lower() + "_" prefix_len = len(prefix) - expand_flags = set( + expand_flags = { x[prefix_len:] for x in myflags if x[:prefix_len] == prefix - ) + } var_split = use_expand_dict.get(k, "").split() var_split = [x for x in var_split if x in expand_flags] var_split.extend(sorted(expand_flags.difference(var_split))) @@ -3098,7 +3153,6 @@ class config: return "" def _getitem(self, mykey): - if mykey in self._constant_keys: # These two point to temporary values when # portage plans to update itself. @@ -3124,7 +3178,7 @@ class config: return ":".join(value) if mykey == "PORTAGE_GID": - return "%s" % portage_gid + return f"{portage_gid}" for d in self.lookuplist: try: @@ -3197,8 +3251,7 @@ class config: "set a value; will be thrown away at reset() time" if not isinstance(myvalue, str): raise ValueError( - "Invalid type being used as a value: '%s': '%s'" - % (str(mykey), str(myvalue)) + f"Invalid type being used as a value: '{str(mykey)}': '{str(myvalue)}'" ) # Avoid potential UnicodeDecodeError exceptions later. @@ -3289,17 +3342,12 @@ class config: if not (src_like_phase and eapi_attrs.sysroot): mydict.pop("ESYSROOT", None) - if not (src_like_phase and eapi_attrs.broot): + if not eapi_attrs.broot: mydict.pop("BROOT", None) - # Prefix variables are supported beginning with EAPI 3, or when - # force-prefix is in FEATURES, since older EAPIs would otherwise be - # useless with prefix configurations. This brings compatibility with - # the prefix branch of portage, which also supports EPREFIX for all - # EAPIs (for obvious reasons). if phase == "depend" or ( - "force-prefix" not in self.features - and eapi is not None + # Prefix variables are supported beginning with EAPI 3. + eapi is not None and not eapi_supports_prefix(eapi) ): mydict.pop("ED", None) @@ -3307,16 +3355,12 @@ class config: mydict.pop("EROOT", None) mydict.pop("ESYSROOT", None) - if ( - phase - not in ( - "pretend", - "setup", - "preinst", - "postinst", - ) - or not eapi_exports_replace_vars(eapi) - ): + if phase not in ( + "pretend", + "setup", + "preinst", + "postinst", + ) or not eapi_exports_replace_vars(eapi): mydict.pop("REPLACING_VERSIONS", None) if phase not in ("prerm", "postrm") or not eapi_exports_replace_vars(eapi): @@ -3328,20 +3372,17 @@ class config: mydict["EBUILD_PHASE_FUNC"] = phase_func if eapi_attrs.posixish_locale: - split_LC_ALL(mydict) - mydict["LC_COLLATE"] = "C" - # check_locale() returns None when check can not be executed. - if check_locale(silent=True, env=mydict) is False: - # try another locale - for l in ("C.UTF-8", "en_US.UTF-8", "en_GB.UTF-8", "C"): - mydict["LC_CTYPE"] = l - if check_locale(silent=True, env=mydict): - # TODO: output the following only once - # writemsg(_("!!! LC_CTYPE unsupported, using %s instead\n") - # % mydict["LC_CTYPE"]) - break - else: - raise AssertionError("C locale did not pass the test!") + if mydict.get("LC_ALL"): + # Sometimes this method is called for processes + # that are not ebuild phases, so only raise + # AssertionError for actual ebuild phases. + if phase and phase not in ("clean", "cleanrm", "fetch"): + raise AssertionError( + f"LC_ALL={mydict['LC_ALL']} for posixish locale. It seems that split_LC_ALL was not called for phase {phase}?" + ) + elif "LC_ALL" in mydict: + # Delete placeholder from split_LC_ALL. + del mydict["LC_ALL"] if not eapi_attrs.exports_PORTDIR: mydict.pop("PORTDIR", None) diff --git a/lib/portage/package/ebuild/deprecated_profile_check.py b/lib/portage/package/ebuild/deprecated_profile_check.py index 19bea1903..ce6476928 100644 --- a/lib/portage/package/ebuild/deprecated_profile_check.py +++ b/lib/portage/package/ebuild/deprecated_profile_check.py @@ -3,7 +3,6 @@ __all__ = ["deprecated_profile_check"] -import io import portage from portage import os, _encodings, _unicode_encode @@ -39,11 +38,10 @@ def deprecated_profile_check(settings=None): if not os.access(deprecated_profile_file, os.R_OK): return - with io.open( + with open( _unicode_encode( deprecated_profile_file, encoding=_encodings["fs"], errors="strict" ), - mode="r", encoding=_encodings["content"], errors="replace", ) as f: diff --git a/lib/portage/package/ebuild/digestcheck.py b/lib/portage/package/ebuild/digestcheck.py index 3fe64550c..cbd57fb58 100644 --- a/lib/portage/package/ebuild/digestcheck.py +++ b/lib/portage/package/ebuild/digestcheck.py @@ -80,7 +80,7 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None): except DigestException as e: eout.eend(1) writemsg(_("\n!!! Digest verification failed:\n"), noiselevel=-1) - writemsg("!!! %s\n" % e.value[0], noiselevel=-1) + writemsg(f"!!! {e.value[0]}\n", noiselevel=-1) writemsg(_("!!! Reason: %s\n") % e.value[1], noiselevel=-1) writemsg(_("!!! Got: %s\n") % e.value[2], noiselevel=-1) writemsg(_("!!! Expected: %s\n") % e.value[3], noiselevel=-1) diff --git a/lib/portage/package/ebuild/digestgen.py b/lib/portage/package/ebuild/digestgen.py index 3a3c92a3a..36d979fff 100644 --- a/lib/portage/package/ebuild/digestgen.py +++ b/lib/portage/package/ebuild/digestgen.py @@ -56,7 +56,7 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None): for myfile in fetchlist_dict[cpv]: distfiles_map.setdefault(myfile, []).append(cpv) except InvalidDependString as e: - writemsg("!!! %s\n" % str(e), noiselevel=-1) + writemsg(f"!!! {str(e)}\n", noiselevel=-1) del e return 0 mytree = os.path.dirname(os.path.dirname(mysettings["O"])) @@ -171,7 +171,7 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None): # digest does not match. cmd = colorize( "INFORM", - "ebuild --force %s manifest" % os.path.basename(myebuild), + f"ebuild --force {os.path.basename(myebuild)} manifest", ) writemsg( ( @@ -181,7 +181,7 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None): ) % myfile ) - + "!!! %s\n" % cmd, + + f"!!! {cmd}\n", noiselevel=-1, ) return 0 @@ -199,7 +199,7 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None): ) return 0 except PortagePackageException as e: - writemsg(("!!! %s\n") % (e,), noiselevel=-1) + writemsg(f"!!! {e}\n", noiselevel=-1) return 0 try: mf.write(sign=False) @@ -227,7 +227,7 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None): pv = pkg_key.split("/")[1] for filename in auto_assumed: if filename in fetchlist: - writemsg_stdout(" %s::%s\n" % (pv, filename)) + writemsg_stdout(f" {pv}::{filename}\n") return 1 finally: portage._doebuild_manifest_exempt_depend -= 1 diff --git a/lib/portage/package/ebuild/doebuild.py b/lib/portage/package/ebuild/doebuild.py index 9650a8444..6691db4e9 100644 --- a/lib/portage/package/ebuild/doebuild.py +++ b/lib/portage/package/ebuild/doebuild.py @@ -1,4 +1,4 @@ -# Copyright 2010-2021 Gentoo Authors +# Copyright 2010-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 __all__ = ["doebuild", "doebuild_environment", "spawn", "spawnebuild"] @@ -7,7 +7,6 @@ import grp import gzip import errno import fnmatch -import io from itertools import chain import logging import os as _os @@ -20,6 +19,7 @@ import sys import tempfile from textwrap import wrap import time +from typing import Union import warnings import zlib @@ -43,6 +43,7 @@ portage.proxy.lazyimport.lazyimport( "portage.util._async.SchedulerInterface:SchedulerInterface", "portage.util._eventloop.global_event_loop:global_event_loop", "portage.util.ExtractKernelVersion:ExtractKernelVersion", + "_emerge.EbuildPhase:_setup_locale", ) from portage import ( @@ -66,6 +67,7 @@ from portage.const import ( INVALID_ENV_FILE, MISC_SH_BINARY, PORTAGE_PYM_PACKAGES, + SUPPORTED_GENTOO_BINPKG_FORMATS, ) from portage.data import portage_gid, portage_uid, secpass, uid, userpriv_groups from portage.dbapi.porttree import _parse_uri_map @@ -76,11 +78,11 @@ from portage.dep import ( paren_enclose, use_reduce, ) +from portage.dep.libc import find_libc_deps from portage.eapi import ( eapi_exports_KV, eapi_exports_merge_type, eapi_exports_replace_vars, - eapi_exports_REPOSITORY, eapi_has_required_use, eapi_has_src_prepare_and_src_configure, eapi_has_pkg_pretend, @@ -111,14 +113,15 @@ from portage.util import ( writemsg_stdout, write_atomic, ) -from portage.util.cpuinfo import get_cpu_count +from portage.util.cpuinfo import get_cpu_count, makeopts_to_job_count from portage.util.lafilefixer import rewrite_lafile from portage.util.compression_probe import _compressors from portage.util.futures import asyncio from portage.util.futures.executor.fork import ForkExecutor from portage.util.path import first_existing from portage.util.socks5 import get_socks5_proxy -from portage.versions import _pkgsplit +from portage.util._dyn_libs.dyn_libs import check_dyn_libs_inconsistent +from portage.versions import _pkgsplit, pkgcmp from _emerge.BinpkgEnvExtractor import BinpkgEnvExtractor from _emerge.EbuildBuildDir import EbuildBuildDir from _emerge.EbuildPhase import EbuildPhase @@ -126,7 +129,6 @@ from _emerge.EbuildSpawnProcess import EbuildSpawnProcess from _emerge.Package import Package from _emerge.RootConfig import RootConfig - _unsandboxed_phases = frozenset( [ "clean", @@ -210,6 +212,7 @@ def _doebuild_spawn(phase, settings, actionmap=None, **kwargs): kwargs["pidns"] = ( "pid-sandbox" in settings.features and phase not in _global_pid_phases ) + kwargs["warn_on_large_env"] = "warn-on-large-env" in settings.features if phase == "depend": kwargs["droppriv"] = "userpriv" in settings.features @@ -228,7 +231,7 @@ def _doebuild_spawn(phase, settings, actionmap=None, **kwargs): else: ebuild_sh_arg = phase - cmd = "%s %s" % ( + cmd = "{} {}".format( _shell_quote( os.path.join( settings["PORTAGE_BIN_PATH"], os.path.basename(EBUILD_SH_BINARY) @@ -237,6 +240,9 @@ def _doebuild_spawn(phase, settings, actionmap=None, **kwargs): ebuild_sh_arg, ) + if phase == "test" and "test_privileged" in settings["PORTAGE_PROPERTIES"].split(): + kwargs["droppriv"] = False + settings["EBUILD_PHASE"] = phase try: return spawn(cmd, settings, **kwargs) @@ -245,17 +251,23 @@ def _doebuild_spawn(phase, settings, actionmap=None, **kwargs): def _spawn_phase( - phase, settings, actionmap=None, returnpid=False, logfile=None, **kwargs + phase, + settings, + actionmap=None, + returnpid=False, + returnproc=False, + logfile=None, + **kwargs, ): - - if returnpid: + if returnproc or returnpid: return _doebuild_spawn( phase, settings, actionmap=actionmap, returnpid=returnpid, + returnproc=returnproc, logfile=logfile, - **kwargs + **kwargs, ) # The logfile argument is unused here, since EbuildPhase uses @@ -266,7 +278,7 @@ def _spawn_phase( phase=phase, scheduler=SchedulerInterface(asyncio._safe_loop()), settings=settings, - **kwargs + **kwargs, ) ebuild_phase.start() @@ -285,20 +297,8 @@ def _doebuild_path(settings, eapi=None): if portage_bin_path[0] != portage.const.PORTAGE_BIN_PATH: # Add a fallback path for restarting failed builds (bug 547086) portage_bin_path.append(portage.const.PORTAGE_BIN_PATH) - prerootpath = [x for x in settings.get("PREROOTPATH", "").split(":") if x] - rootpath = [x for x in settings.get("ROOTPATH", "").split(":") if x] - rootpath_set = frozenset(rootpath) - overrides = [ - x for x in settings.get("__PORTAGE_TEST_PATH_OVERRIDE", "").split(":") if x - ] - prefixes = [] - # settings["EPREFIX"] should take priority over portage.const.EPREFIX - if portage.const.EPREFIX != settings["EPREFIX"] and settings["ROOT"] == os.sep: - prefixes.append(settings["EPREFIX"]) - prefixes.append(portage.const.EPREFIX) - - path = overrides + path = [x for x in settings.get("__PORTAGE_TEST_PATH_OVERRIDE", "").split(":") if x] if "xattr" in settings.features: for x in portage_bin_path: @@ -318,24 +318,20 @@ def _doebuild_path(settings, eapi=None): for x in portage_bin_path: path.append(os.path.join(x, "ebuild-helpers")) - path.extend(prerootpath) - - for prefix in prefixes: - prefix = prefix if prefix else "/" - for x in ( - "usr/local/sbin", - "usr/local/bin", - "usr/sbin", - "usr/bin", - "sbin", - "bin", - ): - # Respect order defined in ROOTPATH - x_abs = os.path.join(prefix, x) - if x_abs not in rootpath_set: - path.append(x_abs) - path.extend(rootpath) + # If PATH is set in env.d, ignore PATH from the calling environment. + # This allows packages to update our PATH as they get installed. + if "PATH" in settings.configdict["env.d"]: + settings.configdict["env"].pop("PATH", None) + + if "PATH" in settings: + pathset = set(path) + for p in settings["PATH"].split(":"): + # Avoid duplicate entries. + if p not in pathset: + path.append(p) + pathset.add(p) + settings["PATH"] = ":".join(path) @@ -460,15 +456,11 @@ def doebuild_environment( mysettings["PN"] = mysplit[0] mysettings["PV"] = mysplit[1] mysettings["PR"] = mysplit[2] + mysettings["PVR"] = mypv[len(mysplit[0]) + 1 :] if noiselimit < 0: mysettings["PORTAGE_QUIET"] = "1" - if mysplit[2] == "r0": - mysettings["PVR"] = mysplit[1] - else: - mysettings["PVR"] = mysplit[1] + "-" + mysplit[2] - # All temporary directories should be subdirectories of # $PORTAGE_TMPDIR/portage, since it's common for /tmp and /var/tmp # to be mounted with the "noexec" option (see bug #346899). @@ -493,7 +485,7 @@ def doebuild_environment( mysettings["SANDBOX_LOG"] = os.path.join(mysettings["T"], "sandbox.log") mysettings["FILESDIR"] = os.path.join(settings["PORTAGE_BUILDDIR"], "files") - # Prefix forward compatability + # Prefix forward compatibility eprefix_lstrip = mysettings["EPREFIX"].lstrip(os.sep) mysettings["ED"] = ( os.path.join(mysettings["D"], eprefix_lstrip).rstrip(os.sep) + os.sep @@ -538,14 +530,6 @@ def doebuild_environment( if not eapi_is_supported(eapi): raise UnsupportedAPIException(mycpv, eapi) - if ( - eapi_exports_REPOSITORY(eapi) - and "PORTAGE_REPO_NAME" in mysettings.configdict["pkg"] - ): - mysettings.configdict["pkg"]["REPOSITORY"] = mysettings.configdict["pkg"][ - "PORTAGE_REPO_NAME" - ] - if mydo != "depend": if hasattr(mydbapi, "getFetchMap") and ( "A" not in mysettings.configdict["pkg"] @@ -618,6 +602,10 @@ def doebuild_environment( nproc = get_cpu_count() if nproc: mysettings["MAKEOPTS"] = "-j%d" % (nproc) + if "GNUMAKEFLAGS" not in mysettings and "MAKEFLAGS" not in mysettings: + mysettings["GNUMAKEFLAGS"] = ( + f"--load-average {nproc} --output-sync=line" + ) if not eapi_exports_KV(eapi): # Discard KV for EAPIs that don't support it. Cached KV is restored @@ -649,27 +637,52 @@ def doebuild_environment( mysettings["KV"] = "" mysettings.backup_changes("KV") + binpkg_format = mysettings.get( + "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0] + ) + if binpkg_format not in portage.const.SUPPORTED_GENTOO_BINPKG_FORMATS: + writemsg( + "!!! BINPKG_FORMAT contains invalid or " + "unsupported format: %s" % binpkg_format, + noiselevel=-1, + ) + binpkg_format = "xpak" + mysettings["BINPKG_FORMAT"] = binpkg_format + binpkg_compression = mysettings.get("BINPKG_COMPRESS", "bzip2") try: compression = _compressors[binpkg_compression] except KeyError as e: if binpkg_compression: writemsg( - "Warning: Invalid or unsupported compression method: %s\n" - % e.args[0] + f"Warning: Invalid or unsupported compression method: {e.args[0]}\n" ) else: # Empty BINPKG_COMPRESS disables compression. mysettings["PORTAGE_COMPRESSION_COMMAND"] = "cat" else: + if ( + settings.get( + f"BINPKG_COMPRESS_FLAGS_{binpkg_compression.upper()}", None + ) + is not None + ): + compression["compress"] = compression["compress"].replace( + "${BINPKG_COMPRESS_FLAGS}", + f"${{BINPKG_COMPRESS_FLAGS_{binpkg_compression.upper()}}}", + ) + try: + compression_binary = compression["compress"].replace( + "{JOBS}", + str(makeopts_to_job_count(mysettings.get("MAKEOPTS", "1"))), + ) compression_binary = shlex_split( - varexpand(compression["compress"], mydict=settings) + varexpand(compression_binary, mydict=settings) )[0] except IndexError as e: writemsg( - "Warning: Invalid or unsupported compression method: %s\n" - % e.args[0] + f"Warning: Invalid or unsupported compression method: {e.args[0]}\n" ) else: if find_binary(compression_binary) is None: @@ -679,9 +692,13 @@ def doebuild_environment( % (binpkg_compression, missing_package) ) else: + compression_binary = compression["compress"].replace( + "{JOBS}", + str(makeopts_to_job_count(mysettings.get("MAKEOPTS", "1"))), + ) cmd = [ varexpand(x, mydict=settings) - for x in shlex_split(compression["compress"]) + for x in shlex_split(compression_binary) ] # Filter empty elements cmd = [x for x in cmd if x != ""] @@ -720,7 +737,8 @@ def doebuild( prev_mtimes=None, fd_pipes=None, returnpid=False, -): + returnproc=False, +) -> Union[int, portage.process.MultiprocessingProcess, list[int]]: """ Wrapper function that invokes specific ebuild phases through the spawning of ebuild.sh @@ -757,9 +775,15 @@ def doebuild( for example. @type fd_pipes: Dictionary @param returnpid: Return a list of process IDs for a successful spawn, or - an integer value if spawn is unsuccessful. NOTE: This requires the - caller clean up all returned PIDs. + an integer value if spawn is unsuccessful. This parameter is supported + supported only when mydo is "depend". NOTE: This requires the caller clean + up all returned PIDs. @type returnpid: Boolean + @param returnproc: Return a MultiprocessingProcess instance for a successful spawn, or + an integer value if spawn is unsuccessful. This parameter is supported + supported only when mydo is "depend". NOTE: This requires the caller to + asynchronously wait for the MultiprocessingProcess instance. + @type returnproc: Boolean @rtype: Boolean @return: 1. 0 for success @@ -852,7 +876,7 @@ def doebuild( if mydo not in validcommands: validcommands.sort() writemsg( - "!!! doebuild: '%s' is not one of the following valid commands:" % mydo, + f"!!! doebuild: '{mydo}' is not one of the following valid commands:", noiselevel=-1, ) for vcount in range(len(validcommands)): @@ -862,26 +886,32 @@ def doebuild( writemsg("\n", noiselevel=-1) return 1 - if returnpid and mydo != "depend": + if (returnproc or returnpid) and mydo != "depend": # This case is not supported, since it bypasses the EbuildPhase class # which implements important functionality (including post phase hooks # and IPC for things like best/has_version and die). + if returnproc: + raise NotImplementedError(f"returnproc not implemented for phase {mydo}") warnings.warn( "portage.doebuild() called " "with returnpid parameter enabled. This usage will " "not be supported in the future.", - DeprecationWarning, + UserWarning, stacklevel=2, ) + elif returnpid: + warnings.warn( + "The portage.doebuild() returnpid parameter is deprecated and replaced by returnproc", + UserWarning, + stacklevel=1, + ) if mydo == "fetchall": fetchall = 1 mydo = "fetch" if mydo not in clean_phases and not os.path.exists(myebuild): - writemsg( - "!!! doebuild: %s not found for %s\n" % (myebuild, mydo), noiselevel=-1 - ) + writemsg(f"!!! doebuild: {myebuild} not found for {mydo}\n", noiselevel=-1) return 1 global _doebuild_manifest_cache @@ -951,7 +981,7 @@ def doebuild( except DigestException as e: out = portage.output.EOutput() out.eerror(_("Digest verification failed:")) - out.eerror("%s" % e.value[0]) + out.eerror(f"{e.value[0]}") out.eerror(_("Reason: %s") % e.value[1]) out.eerror(_("Got: %s") % e.value[2]) out.eerror(_("Expected: %s") % e.value[3]) @@ -962,7 +992,6 @@ def doebuild( return 1 if mf is not _doebuild_manifest_cache and not mf.allow_missing: - # Make sure that all of the ebuilds are # actually listed in the Manifest. for f in os.listdir(pkgdir): @@ -1006,6 +1035,13 @@ def doebuild( myebuild, mydo, myroot, mysettings, debug, use_cache, mydbapi ) + # For returnproc or returnpid assume that the event loop is running + # so we can't run the event loop to call _setup_locale in this case + # and we have to assume the caller took care of it (otherwise + # config.environ() will raise AssertionError). + if not (returnproc or returnpid): + asyncio.run(_setup_locale(mysettings)) + if mydo in clean_phases: builddir_lock = None if not returnpid and "PORTAGE_BUILDDIR_LOCKED" not in mysettings: @@ -1025,14 +1061,17 @@ def doebuild( # get possible slot information from the deps file if mydo == "depend": - if not returnpid: - raise TypeError("returnpid must be True for depend phase") + if not (returnproc or returnpid): + raise TypeError("returnproc or returnpid must be True for depend phase") return _spawn_phase( - mydo, mysettings, fd_pipes=fd_pipes, returnpid=returnpid + mydo, + mysettings, + fd_pipes=fd_pipes, + returnpid=returnpid, + returnproc=returnproc, ) if mydo == "nofetch": - if returnpid: writemsg( "!!! doebuild: %s\n" @@ -1045,7 +1084,6 @@ def doebuild( ) if tree == "porttree": - if not returnpid: # Validate dependency metadata here to ensure that ebuilds with # invalid data are never installed via the ebuild command. Skip @@ -1121,7 +1159,7 @@ def doebuild( newstuff = True else: for x in alist: - writemsg_stdout(">>> Checking %s's mtime...\n" % x) + writemsg_stdout(f">>> Checking {x}'s mtime...\n") try: x_st = os.stat(os.path.join(mysettings["DISTDIR"], x)) except OSError: @@ -1224,12 +1262,12 @@ def doebuild( else: vardb = vartree.dbapi cpv = mysettings.mycpv - cpv_slot = "%s%s%s" % (cpv.cp, portage.dep._slot_separator, cpv.slot) + cpv_slot = f"{cpv.cp}{portage.dep._slot_separator}{cpv.slot}" mysettings["REPLACING_VERSIONS"] = " ".join( - set( + { portage.versions.cpv_getversion(match) for match in vardb.match(cpv_slot) + vardb.match("=" + cpv) - ) + } ) # if any of these are being called, handle them -- running them out of @@ -1276,7 +1314,6 @@ def doebuild( ) ) if need_distfiles: - src_uri = mysettings.configdict["pkg"].get("SRC_URI") if src_uri is None: (src_uri,) = mydbapi.aux_get( @@ -1293,7 +1330,7 @@ def doebuild( alist = _parse_uri_map(mysettings.mycpv, metadata, use=use) aalist = _parse_uri_map(mysettings.mycpv, metadata) except InvalidDependString as e: - writemsg("!!! %s\n" % str(e), noiselevel=-1) + writemsg(f"!!! {str(e)}\n", noiselevel=-1) writemsg(_("!!! Invalid SRC_URI for '%s'.\n") % mycpv, noiselevel=-1) del e return 1 @@ -1307,47 +1344,21 @@ def doebuild( if mf is not None: dist_digests = mf.getTypeDigests("DIST") - def _fetch_subprocess(fetchme, mysettings, listonly, dist_digests): - # For userfetch, drop privileges for the entire fetch call, in - # order to handle DISTDIR on NFS with root_squash for bug 601252. - if _want_userfetch(mysettings): - _drop_privs_userfetch(mysettings) - - return fetch( - fetchme, - mysettings, - listonly=listonly, - fetchonly=fetchonly, - allow_missing_digests=False, - digests=dist_digests, - ) - loop = asyncio._safe_loop() - if loop.is_running(): - # Called by EbuildFetchonly for emerge --pretend --fetchonly. - success = fetch( + success = loop.run_until_complete( + loop.run_in_executor( + ForkExecutor(loop=loop), + _fetch_subprocess, fetchme, mysettings, - listonly=listonly, - fetchonly=fetchonly, - allow_missing_digests=False, - digests=dist_digests, - ) - else: - success = loop.run_until_complete( - loop.run_in_executor( - ForkExecutor(loop=loop), - _fetch_subprocess, - fetchme, - mysettings, - listonly, - dist_digests, - ) + listonly, + dist_digests, + fetchonly, ) + ) if not success: # Since listonly mode is called by emerge --pretend in an - # asynchronous context, spawn_nofetch would trigger event loop - # recursion here, therefore delegate execution of pkg_nofetch + # asynchronous context, execution of pkg_nofetch is delegated # to the caller (bug 657360). if not listonly: spawn_nofetch( @@ -1442,7 +1453,7 @@ def doebuild( ) portage.util.ensure_dirs(parent_dir) if not os.access(parent_dir, os.W_OK): - raise PermissionDenied("access('%s', os.W_OK)" % parent_dir) + raise PermissionDenied(f"access('{parent_dir}', os.W_OK)") retval = spawnebuild( mydo, actionmap, @@ -1459,19 +1470,20 @@ def doebuild( if retval == os.EX_OK: if mydo == "package" and bintree is not None: pkg = bintree.inject( - mysettings.mycpv, filename=mysettings["PORTAGE_BINPKG_TMPFILE"] + mysettings.mycpv, + current_pkg_path=mysettings["PORTAGE_BINPKG_TMPFILE"], ) if pkg is not None: infoloc = os.path.join( mysettings["PORTAGE_BUILDDIR"], "build-info" ) build_info = { - "BINPKGMD5": "%s\n" % pkg._metadata["MD5"], + "BINPKGMD5": f"{pkg._metadata['MD5']}\n", } if pkg.build_id is not None: - build_info["BUILD_ID"] = "%s\n" % pkg.build_id + build_info["BUILD_ID"] = f"{pkg.build_id}\n" for k, v in build_info.items(): - with io.open( + with open( _unicode_encode( os.path.join(infoloc, k), encoding=_encodings["fs"], @@ -1569,7 +1581,6 @@ def doebuild( return retval finally: - if builddir_lock is not None: builddir_lock.scheduler.run_until_complete(builddir_lock.async_unlock()) if tmpdir: @@ -1591,6 +1602,22 @@ def doebuild( portage._doebuild_manifest_exempt_depend -= 1 +def _fetch_subprocess(fetchme, mysettings, listonly, dist_digests, fetchonly): + # For userfetch, drop privileges for the entire fetch call, in + # order to handle DISTDIR on NFS with root_squash for bug 601252. + if _want_userfetch(mysettings): + _drop_privs_userfetch(mysettings) + + return fetch( + fetchme, + mysettings, + listonly=listonly, + fetchonly=fetchonly, + allow_missing_digests=False, + digests=dist_digests, + ) + + def _check_temp_dir(settings): if "PORTAGE_TMPDIR" not in settings or not os.path.isdir( settings["PORTAGE_TMPDIR"] @@ -1612,7 +1639,22 @@ def _check_temp_dir(settings): # for those people. checkdir = first_existing(os.path.join(settings["PORTAGE_TMPDIR"], "portage")) - if not os.access(checkdir, os.W_OK): + try: + with tempfile.NamedTemporaryFile(prefix="exectest-", dir=checkdir) as fd: + os.chmod(fd.name, 0o755) + if not os.access(fd.name, os.X_OK): + writemsg( + _( + "Can not execute files in %s\n" + "Likely cause is that you've mounted it with one of the\n" + "following mount options: 'noexec', 'user', 'users'\n\n" + "Please make sure that portage can execute files in this directory.\n" + ) + % checkdir, + noiselevel=-1, + ) + return 1 + except PermissionError: writemsg( _( "%s is not writable.\n" @@ -1623,21 +1665,6 @@ def _check_temp_dir(settings): ) return 1 - with tempfile.NamedTemporaryFile(prefix="exectest-", dir=checkdir) as fd: - os.chmod(fd.name, 0o755) - if not os.access(fd.name, os.X_OK): - writemsg( - _( - "Can not execute files in %s\n" - "Likely cause is that you've mounted it with one of the\n" - "following mount options: 'noexec', 'user', 'users'\n\n" - "Please make sure that portage can execute files in this directory.\n" - ) - % checkdir, - noiselevel=-1, - ) - return 1 - return os.EX_OK @@ -1783,10 +1810,17 @@ def _spawn_actionmap(settings): def _validate_deps(mysettings, myroot, mydo, mydbapi): - - invalid_dep_exempt_phases = set(["clean", "cleanrm", "help", "prerm", "postrm"]) + invalid_dep_exempt_phases = {"clean", "cleanrm", "help", "prerm", "postrm"} all_keys = set(Package.metadata_keys) all_keys.add("SRC_URI") + # Since configdict["pkg"]["USE"] may contain package.use settings + # from config.setcpv, it is inappropriate to use here (bug 675748), + # so discard it. This is only an issue because configdict["pkg"] is + # a sub-optimal place to extract metadata from. This issue does not + # necessarily indicate a flaw in the Package constructor, since + # passing in precalculated USE can be valid for things like + # autounmask USE changes. + all_keys.discard("USE") all_keys = tuple(all_keys) metadata = mysettings.configdict["pkg"] if all(k in metadata for k in ("PORTAGE_REPO_NAME", "SRC_URI")): @@ -1812,6 +1846,10 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi): root_config = RootConfig(mysettings, {"porttree": FakeTree(mydbapi)}, None) + # A USE calculation from setcpv should always be available here because + # mysettings.mycpv is not None, so use it to prevent redundant setcpv calls. + metadata["USE"] = mysettings["PORTAGE_USE"] + pkg = Package( built=False, cpv=mysettings.mycpv, @@ -1824,7 +1862,7 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi): if pkg.invalid: for k, v in pkg.invalid.items(): for msg in v: - msgs.append(" %s\n" % (msg,)) + msgs.append(f" {msg}\n") if msgs: portage.util.writemsg_level( @@ -1858,7 +1896,7 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi): ), noiselevel=-1, ) - writemsg(" %s\n" % reduced_noise, noiselevel=-1) + writemsg(f" {reduced_noise}\n", noiselevel=-1) normalized_required_use = " ".join(pkg._metadata["REQUIRED_USE"].split()) if reduced_noise != normalized_required_use: writemsg( @@ -1870,7 +1908,7 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi): noiselevel=-1, ) writemsg( - " %s\n" % human_readable_required_use(normalized_required_use), + f" {human_readable_required_use(normalized_required_use)}\n", noiselevel=-1, ) writemsg("\n", noiselevel=-1) @@ -1895,7 +1933,7 @@ def spawn( ipc=True, mountns=False, pidns=False, - **keywords + **keywords, ): """ Spawn a subprocess with extra portage-specific options. @@ -1904,7 +1942,7 @@ def spawn( Sandbox: Sandbox means the spawned process will be limited in its ability t read and write files (normally this means it is restricted to ${D}/) SElinux Sandbox: Enables sandboxing on SElinux - Reduced Privileges: Drops privilages such that the process runs as portage:portage + Reduced Privileges: Drops privileges such that the process runs as portage:portage instead of as root. Notes: os.system cannot be used because it messes with signal handling. Instead we @@ -2067,9 +2105,9 @@ def spawn( free = True if mysettings.mycpv is not None: - keywords["opt_name"] = "[%s]" % mysettings.mycpv + keywords["opt_name"] = f"[{mysettings.mycpv}]" else: - keywords["opt_name"] = "[%s/%s]" % ( + keywords["opt_name"] = "[{}/{}]".format( mysettings.get("CATEGORY", ""), mysettings.get("PF", ""), ) @@ -2094,7 +2132,7 @@ def spawn( mysettings.configdict["env"]["LOGNAME"] = logname try: - if keywords.get("returnpid"): + if keywords.get("returnpid") or keywords.get("returnproc"): return spawn_func(mystring, env=mysettings.environ(), **keywords) proc = EbuildSpawnProcess( @@ -2103,7 +2141,7 @@ def spawn( scheduler=SchedulerInterface(asyncio._safe_loop()), spawn_func=spawn_func, settings=mysettings, - **keywords + **keywords, ) proc.start() @@ -2133,7 +2171,6 @@ def spawnebuild( fd_pipes=None, returnpid=False, ): - if returnpid: warnings.warn( "portage.spawnebuild() called " @@ -2171,7 +2208,7 @@ def spawnebuild( if not (mydo == "install" and "noauto" in mysettings.features): check_file = os.path.join( - mysettings["PORTAGE_BUILDDIR"], ".%sed" % mydo.rstrip("e") + mysettings["PORTAGE_BUILDDIR"], f".{mydo.rstrip('e')}ed" ) if os.path.exists(check_file): writemsg_stdout( @@ -2262,7 +2299,7 @@ def _check_build_log(mysettings, out=None): _unicode_encode(logfile, encoding=_encodings["fs"], errors="strict"), mode="rb", ) - except EnvironmentError: + except OSError: return f_real = None @@ -2271,11 +2308,11 @@ def _check_build_log(mysettings, out=None): f = gzip.GzipFile(filename="", mode="rb", fileobj=f) am_maintainer_mode = [] - bash_command_not_found = [] + command_not_found = [] bash_command_not_found_re = re.compile( r"(.*): line (\d*): (.*): command not found$" ) - command_not_found_exclude_re = re.compile(r"/configure: line ") + dash_command_not_found_re = re.compile(r"(.*): (\d+): (.*): not found$") helper_missing_file = [] helper_missing_file_re = re.compile(r"^!!! (do|new).*: .* does not exist$") @@ -2286,7 +2323,7 @@ def _check_build_log(mysettings, out=None): qa_configure_opts = "" try: - with io.open( + with open( _unicode_encode( os.path.join( mysettings["PORTAGE_BUILDDIR"], "build-info", "QA_CONFIGURE_OPTIONS" @@ -2294,27 +2331,26 @@ def _check_build_log(mysettings, out=None): encoding=_encodings["fs"], errors="strict", ), - mode="r", encoding=_encodings["repo.content"], errors="replace", ) as qa_configure_opts_f: qa_configure_opts = qa_configure_opts_f.read() - except IOError as e: + except OSError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): raise qa_configure_opts = qa_configure_opts.split() if qa_configure_opts: if len(qa_configure_opts) > 1: - qa_configure_opts = "|".join("(%s)" % x for x in qa_configure_opts) - qa_configure_opts = "^(%s)$" % qa_configure_opts + qa_configure_opts = "|".join(f"({x})" for x in qa_configure_opts) + qa_configure_opts = f"^({qa_configure_opts})$" else: - qa_configure_opts = "^%s$" % qa_configure_opts[0] + qa_configure_opts = f"^{qa_configure_opts[0]}$" qa_configure_opts = re.compile(qa_configure_opts) qa_am_maintainer_mode = [] try: - with io.open( + with open( _unicode_encode( os.path.join( mysettings["PORTAGE_BUILDDIR"], @@ -2324,23 +2360,22 @@ def _check_build_log(mysettings, out=None): encoding=_encodings["fs"], errors="strict", ), - mode="r", encoding=_encodings["repo.content"], errors="replace", ) as qa_am_maintainer_mode_f: qa_am_maintainer_mode = [ x for x in qa_am_maintainer_mode_f.read().splitlines() if x ] - except IOError as e: + except OSError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): raise if qa_am_maintainer_mode: if len(qa_am_maintainer_mode) > 1: - qa_am_maintainer_mode = "|".join("(%s)" % x for x in qa_am_maintainer_mode) - qa_am_maintainer_mode = "^(%s)$" % qa_am_maintainer_mode + qa_am_maintainer_mode = "|".join(f"({x})" for x in qa_am_maintainer_mode) + qa_am_maintainer_mode = f"^({qa_am_maintainer_mode})$" else: - qa_am_maintainer_mode = "^%s$" % qa_am_maintainer_mode[0] + qa_am_maintainer_mode = f"^{qa_am_maintainer_mode[0]}$" qa_am_maintainer_mode = re.compile(qa_am_maintainer_mode) # Exclude output from dev-libs/yaz-3.0.47 which looks like this: @@ -2359,7 +2394,10 @@ def _check_build_log(mysettings, out=None): setuptools_warn = set() setuptools_warn_re = re.compile(r".*\/setuptools\/.*: .*Warning: (.*)") # skip useless version normalization warnings - setuptools_warn_ignore_re = [re.compile(r"Normalizing .*")] + setuptools_warn_ignore_re = [ + re.compile(r"Normalizing .*"), + re.compile(r"setup.py install is deprecated"), + ] def _eerror(lines): for line in lines: @@ -2378,11 +2416,11 @@ def _check_build_log(mysettings, out=None): ): am_maintainer_mode.append(line.rstrip("\n")) - if ( - bash_command_not_found_re.match(line) is not None - and command_not_found_exclude_re.search(line) is None - ): - bash_command_not_found.append(line.rstrip("\n")) + if bash_command_not_found_re.match(line) is not None: + command_not_found.append(line.rstrip("\n")) + + if dash_command_not_found_re.match(line) is not None: + command_not_found.append(line.rstrip("\n")) if helper_missing_file_re.match(line) is not None: helper_missing_file.append(line.rstrip("\n")) @@ -2408,8 +2446,8 @@ def _check_build_log(mysettings, out=None): except (EOFError, zlib.error) as e: _eerror( [ - "portage encountered a zlib error: '%s'" % (e,), - "while reading the log file: '%s'" % logfile, + f"portage encountered a zlib error: '{e}'", + f"while reading the log file: '{logfile}'", ] ) finally: @@ -2444,10 +2482,10 @@ def _check_build_log(mysettings, out=None): ) _eqawarn(msg) - if bash_command_not_found: + if command_not_found: msg = [_("QA Notice: command not found:")] msg.append("") - msg.extend("\t" + line for line in bash_command_not_found) + msg.extend("\t" + line for line in command_not_found) _eqawarn(msg) if helper_missing_file: @@ -2459,7 +2497,7 @@ def _check_build_log(mysettings, out=None): if configure_opts_warn: msg = [_("QA Notice: Unrecognized configure options:")] msg.append("") - msg.extend("\t%s" % x for x in configure_opts_warn) + msg.extend(f"\t{x}" for x in configure_opts_warn) _eqawarn(msg) if make_jobserver: @@ -2488,8 +2526,8 @@ def _post_src_install_write_metadata(settings): """ eapi_attrs = _get_eapi_attrs(settings.configdict["pkg"]["EAPI"]) - build_info_dir = os.path.join(settings["PORTAGE_BUILDDIR"], "build-info") + metadata_buffer = {} metadata_keys = ["IUSE"] if eapi_attrs.iuse_effective: @@ -2498,14 +2536,14 @@ def _post_src_install_write_metadata(settings): for k in metadata_keys: v = settings.configdict["pkg"].get(k) if v is not None: - write_atomic(os.path.join(build_info_dir, k), v + "\n") + metadata_buffer[k] = v for k in ("CHOST",): v = settings.get(k) if v is not None: - write_atomic(os.path.join(build_info_dir, k), v + "\n") + metadata_buffer[k] = v - with io.open( + with open( _unicode_encode( os.path.join(build_info_dir, "BUILD_TIME"), encoding=_encodings["fs"], @@ -2515,7 +2553,7 @@ def _post_src_install_write_metadata(settings): encoding=_encodings["repo.content"], errors="strict", ) as f: - f.write("%.0f\n" % (time.time(),)) + f.write(f"{time.time():.0f}\n") use = frozenset(settings["PORTAGE_USE"].split()) for k in _vdb_use_conditional_keys: @@ -2543,17 +2581,7 @@ def _post_src_install_write_metadata(settings): except OSError: pass continue - with io.open( - _unicode_encode( - os.path.join(build_info_dir, k), - encoding=_encodings["fs"], - errors="strict", - ), - mode="w", - encoding=_encodings["repo.content"], - errors="strict", - ) as f: - f.write("%s\n" % v) + metadata_buffer[k] = v if eapi_attrs.slot_operator: deps = evaluate_slot_operator_equal_deps(settings, use, QueryCommand.get_db()) @@ -2565,17 +2593,20 @@ def _post_src_install_write_metadata(settings): except OSError: pass continue - with io.open( - _unicode_encode( - os.path.join(build_info_dir, k), - encoding=_encodings["fs"], - errors="strict", - ), - mode="w", - encoding=_encodings["repo.content"], + + metadata_buffer[k] = v + + for k, v in metadata_buffer.items(): + with open( + _unicode_encode( + os.path.join(build_info_dir, k), + encoding=_encodings["fs"], errors="strict", - ) as f: - f.write("%s\n" % v) + ), + mode="w", + encoding=_encodings["repo.content"], + ) as f: + f.write(f"{v}\n") def _preinst_bsdflags(mysettings): @@ -2595,8 +2626,7 @@ def _preinst_bsdflags(mysettings): % (_shell_quote(mysettings["D"]),) ) os.system( - "chflags -R nosunlnk,nouunlnk %s 2>/dev/null" - % (_shell_quote(mysettings["D"]),) + f"chflags -R nosunlnk,nouunlnk {_shell_quote(mysettings['D'])} 2>/dev/null" ) @@ -2639,7 +2669,7 @@ def _post_src_install_uid_fix(mysettings, out): qa_desktop_file = "" try: - with io.open( + with open( _unicode_encode( os.path.join( mysettings["PORTAGE_BUILDDIR"], "build-info", "QA_DESKTOP_FILE" @@ -2647,26 +2677,24 @@ def _post_src_install_uid_fix(mysettings, out): encoding=_encodings["fs"], errors="strict", ), - mode="r", encoding=_encodings["repo.content"], errors="replace", ) as f: qa_desktop_file = f.read() - except IOError as e: + except OSError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): raise qa_desktop_file = qa_desktop_file.split() if qa_desktop_file: if len(qa_desktop_file) > 1: - qa_desktop_file = "|".join("(%s)" % x for x in qa_desktop_file) - qa_desktop_file = "^(%s)$" % qa_desktop_file + qa_desktop_file = "|".join(f"({x})" for x in qa_desktop_file) + qa_desktop_file = f"^({qa_desktop_file})$" else: - qa_desktop_file = "^%s$" % qa_desktop_file[0] + qa_desktop_file = f"^{qa_desktop_file[0]}$" qa_desktop_file = re.compile(qa_desktop_file) while True: - unicode_error = False size = 0 counted_inodes = set() @@ -2675,6 +2703,10 @@ def _post_src_install_uid_fix(mysettings, out): desktopfile_errors = [] for parent, dirs, files in os.walk(destdir): + if portage.utf8_mode: + parent = os.fsencode(parent) + dirs = [os.fsencode(value) for value in dirs] + files = [os.fsencode(value) for value in files] try: parent = _unicode_decode( parent, encoding=_encodings["merge"], errors="strict" @@ -2731,7 +2763,6 @@ def _post_src_install_uid_fix(mysettings, out): is not None ) ): - desktop_validate = validate_desktop_entry(fpath) if desktop_validate: desktopfile_errors.extend(desktop_validate) @@ -2763,7 +2794,7 @@ def _post_src_install_uid_fix(mysettings, out): " %s is not a valid libtool archive, skipping\n" % fpath[len(destdir) :] ) - qa_msg = "QA Notice: invalid .la file found: %s, %s" % ( + qa_msg = "QA Notice: invalid .la file found: {}, {}".format( fpath[len(destdir) :], e, ) @@ -2775,13 +2806,17 @@ def _post_src_install_uid_fix(mysettings, out): if not fixlafiles_announced: fixlafiles_announced = True writemsg("Fixing .la files\n", fd=out) - writemsg(" %s\n" % fpath[len(destdir) :], fd=out) + writemsg(f" {fpath[len(destdir):]}\n", fd=out) # write_atomic succeeds even in some cases in which # a normal write might fail due to file permission # settings on some operating systems such as HP-UX write_atomic( - _unicode_encode( - fpath, encoding=_encodings["merge"], errors="strict" + ( + fpath + if portage.utf8_mode + else _unicode_encode( + fpath, encoding=_encodings["merge"], errors="strict" + ) ), new_contents, mode="wb", @@ -2825,7 +2860,7 @@ def _post_src_install_uid_fix(mysettings, out): build_info_dir = os.path.join(mysettings["PORTAGE_BUILDDIR"], "build-info") - f = io.open( + f = open( _unicode_encode( os.path.join(build_info_dir, "SIZE"), encoding=_encodings["fs"], @@ -2855,6 +2890,48 @@ def _reapply_bsdflags_to_image(mysettings): ) +def _inject_libc_dep(build_info_dir, mysettings): + # + # We could skip this for non-binpkgs but there doesn't seem to be much + # value in that, as users shouldn't downgrade libc anyway. + injected_libc_depstring = [] + for libc_realized_atom in find_libc_deps( + QueryCommand.get_db()[mysettings["EROOT"]]["vartree"].dbapi, True + ): + if pkgcmp(mysettings.mycpv, libc_realized_atom) is not None: + # We don't want to inject deps on ourselves (libc) + injected_libc_depstring = [] + break + + injected_libc_depstring.append(f">={libc_realized_atom}") + + rdepend_file = os.path.join(build_info_dir, "RDEPEND") + # Slurp the existing contents because we need to mangle it a bit + # It'll look something like (if it exists): + # ``` + # app-misc/foo dev-libs/bar + # <newline> + # ```` + rdepend = None + if os.path.exists(rdepend_file): + with open(rdepend_file, encoding="utf-8") as f: + rdepend = f.readlines() + rdepend = "\n".join(rdepend).strip() + + # For RDEPEND, we want an implicit dependency on >=${PROVIDER_OF_LIBC} + # to avoid runtime breakage when merging binpkgs, see bug #753500. + # + if injected_libc_depstring: + if rdepend: + rdepend += f" {' '.join(injected_libc_depstring).strip()}" + else: + # The package doesn't have an RDEPEND, so make one up. + rdepend = " ".join(injected_libc_depstring) + + with open(rdepend_file, "w", encoding="utf-8") as f: + f.write(f"{rdepend}\n") + + def _post_src_install_soname_symlinks(mysettings, out): """ Check that libraries in $D have corresponding soname symlinks. @@ -2864,22 +2941,20 @@ def _post_src_install_soname_symlinks(mysettings, out): """ image_dir = mysettings["D"] - needed_filename = os.path.join( - mysettings["PORTAGE_BUILDDIR"], "build-info", "NEEDED.ELF.2" - ) + build_info_dir = os.path.join(mysettings["PORTAGE_BUILDDIR"], "build-info") + needed_filename = os.path.join(build_info_dir, "NEEDED.ELF.2") f = None try: - f = io.open( + f = open( _unicode_encode( needed_filename, encoding=_encodings["fs"], errors="strict" ), - mode="r", encoding=_encodings["repo.content"], errors="replace", ) lines = f.readlines() - except IOError as e: + except OSError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): raise return @@ -2887,21 +2962,25 @@ def _post_src_install_soname_symlinks(mysettings, out): if f is not None: f.close() + # We do RDEPEND mangling here instead of the natural location + # in _post_src_install_write_metadata because NEEDED hasn't been + # written yet at that point. + _inject_libc_dep(build_info_dir, mysettings) + metadata = {} for k in ("QA_PREBUILT", "QA_SONAME_NO_SYMLINK"): try: - with io.open( + with open( _unicode_encode( os.path.join(mysettings["PORTAGE_BUILDDIR"], "build-info", k), encoding=_encodings["fs"], errors="strict", ), - mode="r", encoding=_encodings["repo.content"], errors="replace", ) as f: v = f.read() - except IOError as e: + except OSError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): raise else: @@ -2919,10 +2998,10 @@ def _post_src_install_soname_symlinks(mysettings, out): qa_soname_no_symlink = metadata.get("QA_SONAME_NO_SYMLINK", "").split() if qa_soname_no_symlink: if len(qa_soname_no_symlink) > 1: - qa_soname_no_symlink = "|".join("(%s)" % x for x in qa_soname_no_symlink) - qa_soname_no_symlink = "^(%s)$" % qa_soname_no_symlink + qa_soname_no_symlink = "|".join(f"({x})" for x in qa_soname_no_symlink) + qa_soname_no_symlink = f"^({qa_soname_no_symlink})$" else: - qa_soname_no_symlink = "^%s$" % qa_soname_no_symlink[0] + qa_soname_no_symlink = f"^{qa_soname_no_symlink[0]}$" qa_soname_no_symlink = re.compile(qa_soname_no_symlink) libpaths = set(portage.util.getlibpaths(mysettings["ROOT"], env=mysettings)) @@ -2962,35 +3041,33 @@ def _post_src_install_soname_symlinks(mysettings, out): build_info_dir = os.path.join(mysettings["PORTAGE_BUILDDIR"], "build-info") try: - with io.open( + with open( _unicode_encode( os.path.join(build_info_dir, "PROVIDES_EXCLUDE"), encoding=_encodings["fs"], errors="strict", ), - mode="r", encoding=_encodings["repo.content"], errors="replace", ) as f: provides_exclude = f.read() - except IOError as e: + except OSError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): raise provides_exclude = "" try: - with io.open( + with open( _unicode_encode( os.path.join(build_info_dir, "REQUIRES_EXCLUDE"), encoding=_encodings["fs"], errors="strict", ), - mode="r", encoding=_encodings["repo.content"], errors="replace", ) as f: requires_exclude = f.read() - except IOError as e: + except OSError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): raise requires_exclude = "" @@ -3013,7 +3090,7 @@ def _post_src_install_soname_symlinks(mysettings, out): entry = NeededEntry.parse(needed_filename, l) except InvalidData as e: portage.util.writemsg_level( - "\n%s\n\n" % (e,), level=logging.ERROR, noiselevel=-1 + f"\n{e}\n\n", level=logging.ERROR, noiselevel=-1 ) continue @@ -3067,7 +3144,7 @@ def _post_src_install_soname_symlinks(mysettings, out): needed_file.close() if soname_deps.requires is not None: - with io.open( + with open( _unicode_encode( os.path.join(build_info_dir, "REQUIRES"), encoding=_encodings["fs"], @@ -3080,7 +3157,7 @@ def _post_src_install_soname_symlinks(mysettings, out): f.write(soname_deps.requires) if soname_deps.provides is not None: - with io.open( + with open( _unicode_encode( os.path.join(build_info_dir, "PROVIDES"), encoding=_encodings["fs"], @@ -3091,11 +3168,19 @@ def _post_src_install_soname_symlinks(mysettings, out): errors="strict", ) as f: f.write(soname_deps.provides) + else: + if check_dyn_libs_inconsistent(image_dir, soname_deps.provides): + eerror( + "Error! Installing dynamic libraries (.so) with blank PROVIDES!", + phase="install", + key=mysettings.mycpv, + out=out, + ) if unrecognized_elf_files: qa_msg = ["QA Notice: Unrecognized ELF file(s):"] qa_msg.append("") - qa_msg.extend("\t%s" % str(entry).rstrip() for entry in unrecognized_elf_files) + qa_msg.extend(f"\t{str(entry).rstrip()}" for entry in unrecognized_elf_files) qa_msg.append("") for line in qa_msg: eqawarn(line, key=mysettings.mycpv, out=out) diff --git a/lib/portage/package/ebuild/fetch.py b/lib/portage/package/ebuild/fetch.py index 8c64362c2..bfa0c2b27 100644 --- a/lib/portage/package/ebuild/fetch.py +++ b/lib/portage/package/ebuild/fetch.py @@ -6,7 +6,6 @@ __all__ = ["fetch"] import errno import functools import glob -import io import itertools import json import logging @@ -135,7 +134,6 @@ def _spawn_fetch(settings, args, **kwargs): # wget pollute stderr (if portage detects a problem then it # can send it's own message to stderr). if "fd_pipes" not in kwargs: - kwargs["fd_pipes"] = { 0: portage._get_stdin().fileno(), 1: sys.__stdout__.fileno(), @@ -234,7 +232,7 @@ def _ensure_distdir(settings, distdir): if "FAKED_MODE" in settings: # When inside fakeroot, directories with portage's gid appear # to have root's gid. Therefore, use root's gid instead of - # portage's gid to avoid spurrious permissions adjustments + # portage's gid to avoid spurious permissions adjustments # when inside fakeroot. dir_gid = 0 @@ -242,20 +240,15 @@ def _ensure_distdir(settings, distdir): userpriv = portage.data.secpass >= 2 and "userpriv" in settings.features write_test_file = os.path.join(distdir, ".__portage_test_write__") - try: - st = os.stat(distdir) - except OSError: - st = None - - if st is not None and stat.S_ISDIR(st.st_mode): - if not (userfetch or userpriv): - return - if _userpriv_test_write_file(settings, write_test_file): - return + if _userpriv_test_write_file(settings, write_test_file): + return _userpriv_test_write_file_cache.pop(write_test_file, None) + + already_exists = os.path.isdir(distdir) + if ensure_dirs(distdir, gid=dir_gid, mode=dirmode, mask=modemask): - if st is None: + if not already_exists: # The directory has just been created # and therefore it must be empty. return @@ -371,9 +364,7 @@ def _check_distfile(filename, digests, eout, show_errors=1, hash_filter=None): if hash_filter is not None: digests = _apply_hash_filter(digests, hash_filter) if _check_digests(filename, digests, show_errors=show_errors): - eout.ebegin( - "%s %s ;-)" % (os.path.basename(filename), " ".join(sorted(digests))) - ) + eout.ebegin(f"{os.path.basename(filename)} {' '.join(sorted(digests))} ;-)") eout.eend(0) else: return (False, st) @@ -579,7 +570,7 @@ class ContentHashLayout(FilenameHashLayout): to a digest value for self.algo, and which can be compared to other DistfileName instances with their digests_equal method. """ - for filename in super(ContentHashLayout, self).get_filenames(distdir): + for filename in super().get_filenames(distdir): yield DistfileName(filename, digests=dict([(self.algo, filename)])) @staticmethod @@ -674,7 +665,7 @@ class MirrorLayoutConfig: ret = [] for val in self.structure: if not self.validate_structure(val): - raise ValueError("Unsupported structure: {}".format(val)) + raise ValueError(f"Unsupported structure: {val}") if val[0] == "flat": ret.append(FlatLayout(*val[1:])) elif val[0] == "filename-hash": @@ -702,9 +693,9 @@ def get_mirror_url(mirror_url, filename, mysettings, cache_path=None): cache = {} if cache_path is not None: try: - with open(cache_path, "r") as f: + with open(cache_path) as f: cache = json.load(f) - except (IOError, ValueError): + except (OSError, ValueError): pass ts, data = cache.get(mirror_url, (0, None)) @@ -712,7 +703,7 @@ def get_mirror_url(mirror_url, filename, mysettings, cache_path=None): if ts >= time.time() - 86400: mirror_conf.deserialize(data) else: - tmpfile = ".layout.conf.%s" % urlparse(mirror_url).hostname + tmpfile = f".layout.conf.{urlparse(mirror_url).hostname}" try: if mirror_url[:1] == "/": tmpfile = os.path.join(mirror_url, "layout.conf") @@ -726,8 +717,8 @@ def get_mirror_url(mirror_url, filename, mysettings, cache_path=None): tmpfile = os.path.join(mysettings["DISTDIR"], tmpfile) mirror_conf.read_from_file(tmpfile) else: - raise IOError() - except (ConfigParserError, IOError, UnicodeDecodeError): + raise OSError() + except (ConfigParserError, OSError, UnicodeDecodeError): pass else: cache[mirror_url] = (time.time(), mirror_conf.serialize()) @@ -989,7 +980,7 @@ def fetch( ] restrict_fetch = "fetch" in restrict - force_mirror = "force-mirror" in features and not restrict_mirror + force_mirror = "force-mirror" in features and not restrict_mirror and try_mirrors file_uri_tuples = [] # Check for 'items' attribute since OrderedDict is not a dict. @@ -1092,7 +1083,7 @@ def fetch( writemsg(_("!!! No known mirror by the name: %s\n") % (mirrorname)) else: writemsg(_("Invalid mirror definition in SRC_URI:\n"), noiselevel=-1) - writemsg(" %s\n" % (myuri), noiselevel=-1) + writemsg(f" {myuri}\n", noiselevel=-1) else: if (restrict_fetch and not override_fetch) or force_mirror: # Only fetch from specific mirrors is allowed. @@ -1131,7 +1122,7 @@ def fetch( _ensure_distdir(mysettings, mysettings["DISTDIR"]) except PortageException as e: if not os.path.isdir(mysettings["DISTDIR"]): - writemsg("!!! %s\n" % str(e), noiselevel=-1) + writemsg(f"!!! {str(e)}\n", noiselevel=-1) writemsg( _("!!! Directory Not Found: DISTDIR='%s'\n") % mysettings["DISTDIR"], @@ -1217,7 +1208,7 @@ def fetch( vfs_stat = os.statvfs(mysettings["DISTDIR"]) except OSError as e: writemsg_level( - "!!! statvfs('%s'): %s\n" % (mysettings["DISTDIR"], e), + f"!!! statvfs('{mysettings['DISTDIR']}'): {e}\n", noiselevel=-1, level=logging.ERROR, ) @@ -1234,7 +1225,6 @@ def fetch( if (size - mysize + vfs_stat.f_bsize) >= ( vfs_stat.f_bsize * vfs_stat.f_bavail ): - if (size - mysize + vfs_stat.f_bsize) >= ( vfs_stat.f_bsize * vfs_stat.f_bfree ): @@ -1248,7 +1238,6 @@ def fetch( has_space = False if distdir_writable and use_locks: - lock_kwargs = {} if fetchonly: lock_kwargs["flags"] = os.O_NONBLOCK @@ -1267,7 +1256,6 @@ def fetch( continue try: if not listonly: - eout = EOutput() eout.quiet = mysettings.get("PORTAGE_QUIET") == "1" match, mystat = _check_distfile( @@ -1444,7 +1432,7 @@ def fetch( shutil.copyfile(mirror_file, download_path) writemsg(_("Local mirror has file: %s\n") % myfile) break - except (IOError, OSError) as e: + except OSError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): raise del e @@ -1482,13 +1470,14 @@ def fetch( if distdir_writable: try: os.unlink(download_path) - except EnvironmentError: + except OSError: pass elif not orig_digests: - # We don't have a digest, but the file exists. We must - # assume that it is fully downloaded. + # We don't have a digest, and the temporary file exists. if not force: - continue + # Try to resume this download when full + # download has not been explicitly forced. + fetched = 1 else: if ( mydigests[myfile].get("size") is not None @@ -1502,7 +1491,7 @@ def fetch( ): eout = EOutput() eout.quiet = mysettings.get("PORTAGE_QUIET") == "1" - eout.ebegin("%s size ;-)" % (myfile,)) + eout.ebegin(f"{myfile} size ;-)") eout.eend(0) continue else: @@ -1553,9 +1542,7 @@ def fetch( if digests: digests = list(digests) digests.sort() - eout.ebegin( - "%s %s ;-)" % (myfile, " ".join(digests)) - ) + eout.ebegin(f"{myfile} {' '.join(digests)} ;-)") eout.eend(0) continue # fetch any remaining files @@ -1575,6 +1562,7 @@ def fetch( tried_locations.add(loc) if listonly: writemsg_stdout(loc + " ", noiselevel=-1) + fetched = 2 continue # allow different fetchcommands per protocol protocol = loc[0 : loc.find("://")] @@ -1734,7 +1722,7 @@ def fetch( try: variables["DIGESTS"] = " ".join( [ - "%s:%s" % (k.lower(), v) + f"{k.lower()}:{v}" for k, v in mydigests[myfile].items() if k != "size" ] @@ -1752,7 +1740,6 @@ def fetch( myret = -1 try: - myret = _spawn_fetch(mysettings, myfetch) finally: @@ -1783,7 +1770,7 @@ def fetch( os.unlink(download_path) fetched = 0 continue - except EnvironmentError: + except OSError: pass if mydigests is not None and myfile in mydigests: @@ -1795,7 +1782,6 @@ def fetch( del e fetched = 0 else: - if stat.S_ISDIR(mystat.st_mode): # This can happen if FETCHCOMMAND erroneously # contains wget's -P option where it should @@ -1849,13 +1835,12 @@ def fetch( "<title>.*(not found|404).*</title>", re.I | re.M, ) - with io.open( + with open( _unicode_encode( download_path, encoding=_encodings["fs"], errors="strict", ), - mode="r", encoding=_encodings["content"], errors="replace", ) as f: @@ -1869,7 +1854,7 @@ def fetch( ) fetched = 0 continue - except (IOError, OSError): + except OSError: pass fetched = 1 continue @@ -1946,8 +1931,7 @@ def fetch( ) if digests: eout.ebegin( - "%s %s ;-)" - % (myfile, " ".join(sorted(digests))) + f"{myfile} {' '.join(sorted(digests))} ;-)" ) eout.eend(0) fetched = 2 @@ -1960,7 +1944,7 @@ def fetch( ) fetched = 2 break - elif mydigests != None: + elif mydigests is not None: writemsg( _("No digest file available and download failed.\n\n"), noiselevel=-1, diff --git a/lib/portage/package/ebuild/getmaskingstatus.py b/lib/portage/package/ebuild/getmaskingstatus.py index b47dd8c50..f4f3e91b3 100644 --- a/lib/portage/package/ebuild/getmaskingstatus.py +++ b/lib/portage/package/ebuild/getmaskingstatus.py @@ -12,7 +12,6 @@ from portage.versions import _pkg_str class _UnmaskHint: - __slots__ = ("key", "value") def __init__(self, key, value): @@ -21,7 +20,6 @@ class _UnmaskHint: class _MaskReason: - __slots__ = ("category", "message", "unmask_hint") def __init__(self, category, message, unmask_hint=None): @@ -43,7 +41,6 @@ def getmaskingstatus(mycpv, settings=None, portdb=None, myrepo=None): def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): - metadata = None installed = False if not isinstance(mycpv, str): @@ -90,9 +87,9 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): properties = metadata["PROPERTIES"] restrict = metadata["RESTRICT"] if not eapi_is_supported(eapi): - return [_MaskReason("EAPI", "EAPI %s" % eapi)] + return [_MaskReason("EAPI", f"EAPI {eapi}")] if _eapi_is_deprecated(eapi) and not installed: - return [_MaskReason("EAPI", "EAPI %s" % eapi)] + return [_MaskReason("EAPI", f"EAPI {eapi}")] egroups = settings.configdict["backupenv"].get("ACCEPT_KEYWORDS", "").split() global_accept_keywords = settings.get("ACCEPT_KEYWORDS", "") pgroups = global_accept_keywords.split() @@ -149,7 +146,7 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): try: missing_licenses = settings._getMissingLicenses(mycpv, metadata) if missing_licenses: - allowed_tokens = set(["||", "(", ")"]) + allowed_tokens = {"||", "(", ")"} allowed_tokens.update(missing_licenses) license_split = licenses.split() license_split = [x for x in license_split if x in allowed_tokens] @@ -168,7 +165,7 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): try: missing_properties = settings._getMissingProperties(mycpv, metadata) if missing_properties: - allowed_tokens = set(["||", "(", ")"]) + allowed_tokens = {"||", "(", ")"} allowed_tokens.update(missing_properties) properties_split = properties.split() properties_split = [x for x in properties_split if x in allowed_tokens] @@ -185,7 +182,7 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): msg.append("in RESTRICT") rValue.append(_MaskReason("RESTRICT", " ".join(msg))) except InvalidDependString as e: - rValue.append(_MaskReason("invalid", "RESTRICT: %s" % (e,))) + rValue.append(_MaskReason("invalid", f"RESTRICT: {e}")) # Only show KEYWORDS masks for installed packages # if they're not masked for any other reason. diff --git a/lib/portage/package/ebuild/meson.build b/lib/portage/package/ebuild/meson.build new file mode 100644 index 000000000..69fb4f588 --- /dev/null +++ b/lib/portage/package/ebuild/meson.build @@ -0,0 +1,23 @@ +py.install_sources( + [ + 'config.py', + 'deprecated_profile_check.py', + 'digestcheck.py', + 'digestgen.py', + 'doebuild.py', + 'fetch.py', + 'getmaskingreason.py', + 'getmaskingstatus.py', + 'prepare_build_dirs.py', + 'profile_iuse.py', + '_metadata_invalid.py', + '_spawn_nofetch.py', + '__init__.py', + ], + subdir : 'portage/package/ebuild', + pure : not native_extensions +) + +subdir('_config') +subdir('_ipc') +subdir('_parallel_manifest') diff --git a/lib/portage/package/ebuild/prepare_build_dirs.py b/lib/portage/package/ebuild/prepare_build_dirs.py index 659198905..9471179aa 100644 --- a/lib/portage/package/ebuild/prepare_build_dirs.py +++ b/lib/portage/package/ebuild/prepare_build_dirs.py @@ -54,7 +54,7 @@ def prepare_build_dirs(myroot=None, settings=None, cleanup=False): if errno.ENOENT == oe.errno: pass elif errno.EPERM == oe.errno: - writemsg("%s\n" % oe, noiselevel=-1) + writemsg(f"{oe}\n", noiselevel=-1) writemsg( _("Operation Not Permitted: rmtree('%s')\n") % clean_dir, noiselevel=-1, @@ -72,7 +72,7 @@ def prepare_build_dirs(myroot=None, settings=None, cleanup=False): if errno.EEXIST == oe.errno: pass elif errno.EPERM == oe.errno: - writemsg("%s\n" % oe, noiselevel=-1) + writemsg(f"{oe}\n", noiselevel=-1) writemsg( _("Operation Not Permitted: makedirs('%s')\n") % dir_path, noiselevel=-1, @@ -102,6 +102,15 @@ def prepare_build_dirs(myroot=None, settings=None, cleanup=False): apply_secpass_permissions( mysettings[dir_key], uid=portage_uid, gid=portage_gid ) + # The setgid bit prevents a lockfile group permission race for bug #468990. + ipc_kwargs = {} + if portage.data.secpass >= 1: + ipc_kwargs["gid"] = portage_gid + ipc_kwargs["mode"] = 0o2770 + ensure_dirs( + os.path.join(mysettings["PORTAGE_BUILDDIR"], ".ipc"), + **ipc_kwargs, + ) except PermissionDenied as e: writemsg(_("Permission Denied: %s\n") % str(e), noiselevel=-1) return 1 @@ -142,7 +151,7 @@ def _adjust_perms_msg(settings, msg): mode="ab", ) log_file_real = log_file - except IOError: + except OSError: def write(msg): pass @@ -165,7 +174,6 @@ def _adjust_perms_msg(settings, msg): def _prepare_features_dirs(mysettings): - # Use default ABI libdir in accordance with bug #355283. libdir = None default_abi = mysettings.get("DEFAULT_ABI") @@ -228,11 +236,9 @@ def _prepare_features_dirs(mysettings): except OSError: continue if subdir_st.st_gid != portage_gid or ( - ( - stat.S_ISDIR(subdir_st.st_mode) - and not dirmode - == (stat.S_IMODE(subdir_st.st_mode) & dirmode) - ) + stat.S_ISDIR(subdir_st.st_mode) + and not dirmode + == (stat.S_IMODE(subdir_st.st_mode) & dirmode) ): droppriv_fix = True break @@ -284,7 +290,7 @@ def _prepare_features_dirs(mysettings): except PortageException as e: failure = True - writemsg("\n!!! %s\n" % str(e), noiselevel=-1) + writemsg(f"\n!!! {str(e)}\n", noiselevel=-1) writemsg( _("!!! Failed resetting perms on %s='%s'\n") % (kwargs["basedir_var"], basedir), @@ -308,7 +314,7 @@ def _prepare_workdir(mysettings): else: raise ValueError() if parsed_mode & 0o7777 != parsed_mode: - raise ValueError("Invalid file mode: %s" % mode) + raise ValueError(f"Invalid file mode: {mode}") else: workdir_mode = parsed_mode except KeyError as e: @@ -317,7 +323,7 @@ def _prepare_workdir(mysettings): ) except ValueError as e: if len(str(e)) > 0: - writemsg("%s\n" % e) + writemsg(f"{e}\n") writemsg( _("!!! Unable to parse PORTAGE_WORKDIR_MODE='%s', using %s.\n") % (mysettings["PORTAGE_WORKDIR_MODE"], oct(workdir_mode)) @@ -355,7 +361,7 @@ def _prepare_workdir(mysettings): mode=0o2770, ) except PortageException as e: - writemsg("!!! %s\n" % str(e), noiselevel=-1) + writemsg(f"!!! {str(e)}\n", noiselevel=-1) writemsg( _("!!! Permission issues with PORTAGE_LOGDIR='%s'\n") % mysettings["PORTAGE_LOGDIR"], @@ -387,7 +393,7 @@ def _prepare_workdir(mysettings): log_subdir = os.path.join(logdir, "build", mysettings["CATEGORY"]) mysettings["PORTAGE_LOG_FILE"] = os.path.join( log_subdir, - "%s:%s.log%s" % (mysettings["PF"], logid_time, compress_log_ext), + f"{mysettings['PF']}:{logid_time}.log{compress_log_ext}", ) else: log_subdir = logdir @@ -408,16 +414,17 @@ def _prepare_workdir(mysettings): try: _ensure_log_subdirs(logdir, log_subdir) except PortageException as e: - writemsg("!!! %s\n" % (e,), noiselevel=-1) + writemsg(f"!!! {e}\n", noiselevel=-1) if os.access(log_subdir, os.W_OK): logdir_subdir_ok = True else: writemsg( - "!!! %s: %s\n" % (_("Permission Denied"), log_subdir), noiselevel=-1 + f"!!! {_('Permission Denied')}: {log_subdir}\n", + noiselevel=-1, ) - tmpdir_log_path = os.path.join(mysettings["T"], "build.log%s" % compress_log_ext) + tmpdir_log_path = os.path.join(mysettings["T"], f"build.log{compress_log_ext}") if not logdir_subdir_ok: # NOTE: When sesandbox is enabled, the local SELinux security policies # may not allow output to be piped out of the sesandbox domain. The |