# Copyright 1999-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import functools import gzip import io import json import sys import tempfile from _emerge.AsynchronousLock import AsynchronousLock from _emerge.BinpkgEnvExtractor import BinpkgEnvExtractor from _emerge.MiscFunctionsProcess import MiscFunctionsProcess from _emerge.EbuildProcess import EbuildProcess from _emerge.CompositeTask import CompositeTask from _emerge.PackagePhase import PackagePhase from _emerge.TaskSequence import TaskSequence from portage.package.ebuild._ipc.QueryCommand import QueryCommand from portage.util._dyn_libs.soname_deps_qa import ( _get_all_provides, _get_unresolved_soname_deps, ) from portage.package.ebuild.prepare_build_dirs import ( _prepare_workdir, _prepare_fake_distdir, _prepare_fake_filesdir, ) from portage.eapi import _get_eapi_attrs from portage.util import writemsg, ensure_dirs from portage.util._async.AsyncTaskFuture import AsyncTaskFuture from portage.util._async.BuildLogger import BuildLogger from portage.util.futures import asyncio from portage.util.futures.executor.fork import ForkExecutor from portage.exception import InvalidBinaryPackageFormat from portage.const import SUPPORTED_GENTOO_BINPKG_FORMATS try: from portage.xml.metadata import MetaDataXML except (SystemExit, KeyboardInterrupt): raise except (ImportError, SystemError, RuntimeError, Exception): # broken or missing xml support # https://bugs.python.org/issue14988 MetaDataXML = None import portage portage.proxy.lazyimport.lazyimport( globals(), "portage.elog:messages@elog_messages", "portage.package.ebuild.doebuild:_check_build_log," + "_post_phase_cmds,_post_phase_userpriv_perms," + "_post_phase_emptydir_cleanup," + "_post_src_install_soname_symlinks," + "_post_src_install_uid_fix,_postinst_bsdflags," + "_post_src_install_write_metadata," + "_preinst_bsdflags", "portage.util.futures.unix_events:_set_nonblocking", "portage.util.locale:async_check_locale,split_LC_ALL", ) from portage import os from portage import _encodings from portage import _unicode_encode async def _setup_locale(settings): eapi_attrs = _get_eapi_attrs(settings["EAPI"]) if eapi_attrs.posixish_locale: split_LC_ALL(settings) settings["LC_COLLATE"] = "C" # check_locale() returns None when check can not be executed. if await async_check_locale(silent=True, env=settings.environ()) is False: # try another locale for l in ("C.UTF-8", "en_US.UTF-8", "en_GB.UTF-8", "C"): settings["LC_CTYPE"] = l if await async_check_locale(silent=True, env=settings.environ()): # TODO: output the following only once # writemsg( # _("!!! LC_CTYPE unsupported, using %s instead\n") # % self.settings["LC_CTYPE"] # ) break else: raise AssertionError("C locale did not pass the test!") async def _setup_repo_revisions(settings): repo_name = settings.configdict["pkg"].get("PORTAGE_REPO_NAME") db = getattr(settings.mycpv, "_db", None) if ( isinstance(db, portage.portdbapi) and repo_name and "PORTAGE_REPO_REVISIONS" not in settings.configdict["pkg"] ): repo = db.repositories[repo_name] ec_dict = repo.eclass_db.get_eclass_data( settings.configdict["pkg"]["INHERITED"].split() ) referenced_repos = {repo.name: repo} for ec_info in ec_dict.values(): ec_repo = db.repositories.get_repo_for_location( os.path.dirname(os.path.dirname(ec_info.location)) ) referenced_repos.setdefault(ec_repo.name, ec_repo) repo_revisions = {} for repo_ref in referenced_repos.values(): if repo_ref.sync_type: sync = portage.sync.module_controller.get_class(repo_ref.sync_type)() try: # TODO: Wait for subprocesses asynchronously here. status, repo_revision = sync.retrieve_head( options={"repo": repo_ref} ) except NotImplementedError: pass else: if status == os.EX_OK: repo_revisions[repo_ref.name] = repo_revision.strip() settings.configdict["pkg"]["PORTAGE_REPO_REVISIONS"] = json.dumps( repo_revisions, ensure_ascii=False, sort_keys=True ) class EbuildPhase(CompositeTask): __slots__ = ("actionmap", "fd_pipes", "phase", "settings") + ("_ebuild_lock",) # FEATURES displayed prior to setup phase _features_display = ( "ccache", "compressdebug", "dedupdebug", "distcc", "fakeroot", "installsources", "keeptemp", "keepwork", "network-sandbox", "network-sandbox-proxy", "nostrip", "preserve-libs", "sandbox", "selinux", "sesandbox", "splitdebug", "suidctl", "test", "userpriv", "usersandbox", ) # Locked phases _locked_phases = ("setup", "preinst", "postinst", "prerm", "postrm") def _start(self): future = asyncio.ensure_future(self._async_start(), loop=self.scheduler) self._start_task(AsyncTaskFuture(future=future), self._async_start_exit) async def _async_start(self): await _setup_locale(self.settings) await _setup_repo_revisions(self.settings) need_builddir = self.phase not in EbuildProcess._phases_without_builddir if need_builddir: phase_completed_file = os.path.join( self.settings["PORTAGE_BUILDDIR"], f".{self.phase.rstrip('e')}ed" ) if not os.path.exists(phase_completed_file): # If the phase is really going to run then we want # to eliminate any stale elog messages that may # exist from a previous run. try: os.unlink(os.path.join(self.settings["T"], "logging", self.phase)) except OSError: pass ensure_dirs(os.path.join(self.settings["PORTAGE_BUILDDIR"], "empty")) if self.phase in ("nofetch", "pretend", "setup"): use = self.settings.get("PORTAGE_BUILT_USE") if use is None: use = self.settings["PORTAGE_USE"] maint_str = "" upstr_str = "" metadata_xml_path = os.path.join( os.path.dirname(self.settings["EBUILD"]), "metadata.xml" ) if MetaDataXML is not None and os.path.isfile(metadata_xml_path): herds_path = os.path.join( self.settings["PORTDIR"], "metadata/herds.xml" ) try: metadata_xml = MetaDataXML(metadata_xml_path, herds_path) maint_str = metadata_xml.format_maintainer_string() upstr_str = metadata_xml.format_upstream_string() except SyntaxError: maint_str = "" msg = [] msg.append(f"Package: {self.settings.mycpv}:{self.settings['SLOT']}") if self.settings.get("PORTAGE_REPO_NAME"): msg.append(f"Repository: {self.settings['PORTAGE_REPO_NAME']}") if maint_str: msg.append(f"Maintainer: {maint_str}") if upstr_str: msg.append(f"Upstream: {upstr_str}") msg.append(f"USE: {use}") relevant_features = [] enabled_features = self.settings.features for x in self._features_display: if x in enabled_features: relevant_features.append(x) if relevant_features: msg.append(f"FEATURES: {' '.join(relevant_features)}") # Force background=True for this header since it's intended # for the log and it doesn't necessarily need to be visible # elsewhere. await self._elog("einfo", msg, background=True) if self.phase == "package": if "PORTAGE_BINPKG_TMPFILE" not in self.settings: binpkg_format = self.settings.get( "BINPKG_FORMAT", SUPPORTED_GENTOO_BINPKG_FORMATS[0] ) if binpkg_format == "xpak": self.settings["BINPKG_FORMAT"] = "xpak" self.settings["PORTAGE_BINPKG_TMPFILE"] = ( os.path.join( self.settings["PKGDIR"], self.settings["CATEGORY"], self.settings["PF"], ) + ".tbz2" ) elif binpkg_format == "gpkg": self.settings["BINPKG_FORMAT"] = "gpkg" self.settings["PORTAGE_BINPKG_TMPFILE"] = ( os.path.join( self.settings["PKGDIR"], self.settings["CATEGORY"], self.settings["PF"], ) + ".gpkg.tar" ) else: raise InvalidBinaryPackageFormat(binpkg_format) self.settings.mycpv._db.bintree._ensure_dir( os.path.dirname(self.settings["PORTAGE_BINPKG_TMPFILE"]) ) def _async_start_exit(self, task): task.future.cancelled() or task.future.result() if self._default_exit(task) != os.EX_OK: self.wait() return if self.phase in ("pretend", "prerm"): env_extractor = BinpkgEnvExtractor( background=self.background, scheduler=self.scheduler, settings=self.settings, ) if env_extractor.saved_env_exists(): self._start_task(env_extractor, self._env_extractor_exit) return # If the environment.bz2 doesn't exist, then ebuild.sh will # source the ebuild as a fallback. self._start_lock() def _env_extractor_exit(self, env_extractor): if self._default_exit(env_extractor) != os.EX_OK: self.wait() return self._start_lock() def _start_lock(self): if ( self.phase in self._locked_phases and "ebuild-locks" in self.settings.features ): eroot = self.settings["EROOT"] lock_path = os.path.join(eroot, portage.VDB_PATH + "-ebuild") if os.access(os.path.dirname(lock_path), os.W_OK): self._ebuild_lock = AsynchronousLock( path=lock_path, scheduler=self.scheduler ) self._start_task(self._ebuild_lock, self._lock_exit) return self._start_ebuild() def _lock_exit(self, ebuild_lock): if self._default_exit(ebuild_lock) != os.EX_OK: self.wait() return self._start_ebuild() def _get_log_path(self): # Don't open the log file during the clean phase since the # open file can result in an nfs lock on $T/build.log which # prevents the clean phase from removing $T. logfile = None if ( self.phase not in ("clean", "cleanrm") and self.settings.get("PORTAGE_BACKGROUND") != "subprocess" ): logfile = self.settings.get("PORTAGE_LOG_FILE") return logfile def _start_ebuild(self): if self.phase == "package": self._start_task( PackagePhase( actionmap=self.actionmap, background=self.background, fd_pipes=self.fd_pipes, logfile=self._get_log_path(), scheduler=self.scheduler, settings=self.settings, ), self._ebuild_exit, ) return if self.phase == "unpack": alist = self.settings.configdict["pkg"].get("A", "").split() _prepare_fake_distdir(self.settings, alist) _prepare_fake_filesdir(self.settings) fd_pipes = self.fd_pipes if fd_pipes is None: if not self.background and self.phase == "nofetch": # All the pkg_nofetch output goes to stderr since # it's considered to be an error message. fd_pipes = {1: sys.__stderr__.fileno()} ebuild_process = EbuildProcess( actionmap=self.actionmap, background=self.background, fd_pipes=fd_pipes, logfile=self._get_log_path(), phase=self.phase, scheduler=self.scheduler, settings=self.settings, ) self._start_task(ebuild_process, self._ebuild_exit) def _ebuild_exit(self, ebuild_process): self._assert_current(ebuild_process) if self._ebuild_lock is None: self._ebuild_exit_unlocked(ebuild_process) else: self._start_task( AsyncTaskFuture(future=self._ebuild_lock.async_unlock()), functools.partial(self._ebuild_exit_unlocked, ebuild_process), ) def _ebuild_exit_unlocked(self, ebuild_process, unlock_task=None): if unlock_task is not None: self._assert_current(unlock_task) if unlock_task.cancelled: self._default_final_exit(unlock_task) return # Normally, async_unlock should not raise an exception here. unlock_task.future.result() fail = False if ebuild_process.returncode != os.EX_OK: self.returncode = ebuild_process.returncode if self.phase == "test" and "test-fail-continue" in self.settings.features: # mark test phase as complete (bug #452030) try: open( _unicode_encode( os.path.join(self.settings["PORTAGE_BUILDDIR"], ".tested"), encoding=_encodings["fs"], errors="strict", ), "wb", ).close() except OSError: pass else: fail = True if not fail: self.returncode = None logfile = self._get_log_path() if self.phase == "install": out = io.StringIO() _check_build_log(self.settings, out=out) msg = out.getvalue() self.scheduler.output(msg, log_path=logfile) if fail: self._die_hooks() return settings = self.settings _post_phase_userpriv_perms(settings) _post_phase_emptydir_cleanup(settings) if self.phase == "unpack": # Bump WORKDIR timestamp, in case tar gave it a timestamp # that will interfere with distfiles / WORKDIR timestamp # comparisons as reported in bug #332217. Also, fix # ownership since tar can change that too. os.utime(settings["WORKDIR"], None) _prepare_workdir(settings) elif self.phase == "install": out = io.StringIO() _post_src_install_write_metadata(settings) _post_src_install_uid_fix(settings, out) msg = out.getvalue() if msg: self.scheduler.output(msg, log_path=logfile) elif self.phase == "preinst": _preinst_bsdflags(settings) elif self.phase == "postinst": _postinst_bsdflags(settings) post_phase_cmds = _post_phase_cmds.get(self.phase) if post_phase_cmds is not None: if logfile is not None and self.phase in ("install",): # Log to a temporary file, since the code we are running # reads PORTAGE_LOG_FILE for QA checks, and we want to # avoid annoying "gzip: unexpected end of file" messages # when FEATURES=compress-build-logs is enabled. fd, logfile = tempfile.mkstemp() os.close(fd) post_phase = _PostPhaseCommands( background=self.background, commands=post_phase_cmds, elog=self._elog, fd_pipes=self.fd_pipes, logfile=logfile, phase=self.phase, scheduler=self.scheduler, settings=settings, ) self._start_task(post_phase, self._post_phase_exit) return # this point is not reachable if there was a failure and # we returned for die_hooks above, so returncode must # indicate success (especially if ebuild_process.returncode # is unsuccessful and test-fail-continue came into play) self.returncode = os.EX_OK self._current_task = None self.wait() def _post_phase_exit(self, post_phase): self._assert_current(post_phase) log_path = None if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": log_path = self.settings.get("PORTAGE_LOG_FILE") if post_phase.logfile is not None and post_phase.logfile != log_path: # We were logging to a temp file (see above), so append # temp file to main log and remove temp file. self._append_temp_log(post_phase.logfile, log_path) if self._final_exit(post_phase) != os.EX_OK: writemsg(f"!!! post {self.phase} failed; exiting.\n", noiselevel=-1) self._die_hooks() return self._current_task = None self.wait() return def _append_temp_log(self, temp_log, log_path): temp_file = open( _unicode_encode(temp_log, encoding=_encodings["fs"], errors="strict"), "rb" ) log_file, log_file_real = self._open_log(log_path) for line in temp_file: log_file.write(line) temp_file.close() log_file.close() if log_file_real is not log_file: log_file_real.close() os.unlink(temp_log) def _open_log(self, log_path): f = open( _unicode_encode(log_path, encoding=_encodings["fs"], errors="strict"), mode="ab", ) f_real = f if log_path.endswith(".gz"): f = gzip.GzipFile(filename="", mode="ab", fileobj=f) return (f, f_real) def _die_hooks(self): self.returncode = None phase = "die_hooks" die_hooks = MiscFunctionsProcess( background=self.background, commands=[phase], phase=phase, logfile=self._get_log_path(), fd_pipes=self.fd_pipes, scheduler=self.scheduler, settings=self.settings, ) self._start_task(die_hooks, self._die_hooks_exit) def _die_hooks_exit(self, die_hooks): if ( self.phase != "clean" and "noclean" not in self.settings.features and "fail-clean" in self.settings.features ): self._default_exit(die_hooks) self._fail_clean() return self._final_exit(die_hooks) self.returncode = 1 self.wait() def _fail_clean(self): self.returncode = None portage.elog.elog_process(self.settings.mycpv, self.settings) phase = "clean" clean_phase = EbuildPhase( background=self.background, fd_pipes=self.fd_pipes, phase=phase, scheduler=self.scheduler, settings=self.settings, ) self._start_task(clean_phase, self._fail_clean_exit) def _fail_clean_exit(self, clean_phase): self._final_exit(clean_phase) self.returncode = 1 self.wait() async def _elog(self, elog_funcname, lines, background=None): if background is None: background = self.background out = io.StringIO() phase = self.phase elog_func = getattr(elog_messages, elog_funcname) global_havecolor = portage.output.havecolor try: portage.output.havecolor = self.settings.get( "NOCOLOR", "false" ).lower() in ("no", "false") for line in lines: elog_func(line, phase=phase, key=self.settings.mycpv, out=out) finally: portage.output.havecolor = global_havecolor msg = out.getvalue() if msg: build_logger = None try: log_file = None log_path = None if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": log_path = self.settings.get("PORTAGE_LOG_FILE") if log_path: build_logger = BuildLogger( env=self.settings.environ(), log_path=log_path, log_filter_file=self.settings.get( "PORTAGE_LOG_FILTER_FILE_CMD" ), scheduler=self.scheduler, ) build_logger.start() _set_nonblocking(build_logger.stdin.fileno()) log_file = build_logger.stdin await self.scheduler.async_output( msg, log_file=log_file, background=background ) if build_logger is not None: build_logger.stdin.close() await build_logger.async_wait() except asyncio.CancelledError: if build_logger is not None: build_logger.cancel() raise class _PostPhaseCommands(CompositeTask): __slots__ = ("commands", "elog", "fd_pipes", "logfile", "phase", "settings") def _start(self): if isinstance(self.commands, list): cmds = [({}, self.commands)] else: cmds = list(self.commands) if "selinux" not in self.settings.features: cmds = [ (kwargs, commands) for kwargs, commands in cmds if not kwargs.get("selinux_only") ] tasks = TaskSequence() for kwargs, commands in cmds: # Select args intended for MiscFunctionsProcess. kwargs = {k: v for k, v in kwargs.items() if k in ("ld_preload_sandbox",)} tasks.add( MiscFunctionsProcess( background=self.background, commands=commands, fd_pipes=self.fd_pipes, logfile=self.logfile, phase=self.phase, scheduler=self.scheduler, settings=self.settings, **kwargs, ) ) self._start_task(tasks, self._commands_exit) def _commands_exit(self, task): if self._default_exit(task) != os.EX_OK: self._async_wait() return if self.phase == "install": out = io.StringIO() _post_src_install_soname_symlinks(self.settings, out) msg = out.getvalue() if msg: self.scheduler.output( msg, log_path=self.settings.get("PORTAGE_LOG_FILE") ) if "qa-unresolved-soname-deps" in self.settings.features: # This operates on REQUIRES metadata generated by the above function call. future = asyncio.ensure_future( self._soname_deps_qa(), loop=self.scheduler ) # If an unexpected exception occurs, then this will raise it. future.add_done_callback( lambda future: future.cancelled() or future.result() ) self._start_task( AsyncTaskFuture(future=future), self._default_final_exit ) else: self._default_final_exit(task) else: self._default_final_exit(task) async def _soname_deps_qa(self): vardb = QueryCommand.get_db()[self.settings["EROOT"]]["vartree"].dbapi all_provides = await self.scheduler.run_in_executor( ForkExecutor(loop=self.scheduler), _get_all_provides, vardb ) unresolved = _get_unresolved_soname_deps( os.path.join(self.settings["PORTAGE_BUILDDIR"], "build-info"), all_provides ) if unresolved: unresolved.sort() qa_msg = ["QA Notice: Unresolved soname dependencies:"] qa_msg.append("") qa_msg.extend( f"\t{filename}: {' '.join(sorted(soname_deps))}" for filename, soname_deps in unresolved ) qa_msg.append("") await self.elog("eqawarn", qa_msg)