path: root/lib/_emerge/EbuildPhase.py

# Copyright 1999-2020 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2

from __future__ import unicode_literals

import functools
import gzip
import io
import sys
import tempfile

from _emerge.AsynchronousLock import AsynchronousLock
from _emerge.BinpkgEnvExtractor import BinpkgEnvExtractor
from _emerge.MiscFunctionsProcess import MiscFunctionsProcess
from _emerge.EbuildProcess import EbuildProcess
from _emerge.CompositeTask import CompositeTask
from _emerge.PackagePhase import PackagePhase
from _emerge.TaskSequence import TaskSequence
from portage.package.ebuild._ipc.QueryCommand import QueryCommand
from portage.util._dyn_libs.soname_deps_qa import (
	_get_all_provides,
	_get_unresolved_soname_deps,
)
from portage.package.ebuild.prepare_build_dirs import (_prepare_workdir,
		_prepare_fake_distdir, _prepare_fake_filesdir)
from portage.util.futures.compat_coroutine import coroutine, coroutine_return
from portage.util import writemsg
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
from portage.util._async.BuildLogger import BuildLogger
from portage.util.futures import asyncio
from portage.util.futures.executor.fork import ForkExecutor

try:
	from portage.xml.metadata import MetaDataXML
except (SystemExit, KeyboardInterrupt):
	raise
except (ImportError, SystemError, RuntimeError, Exception):
	# broken or missing xml support
	# https://bugs.python.org/issue14988
	MetaDataXML = None

import portage
portage.proxy.lazyimport.lazyimport(globals(),
	'portage.elog:messages@elog_messages',
	'portage.package.ebuild.doebuild:_check_build_log,' + \
		'_post_phase_cmds,_post_phase_userpriv_perms,' + \
		'_post_src_install_soname_symlinks,' + \
		'_post_src_install_uid_fix,_postinst_bsdflags,' + \
		'_post_src_install_write_metadata,' + \
		'_preinst_bsdflags'
)
from portage import os
from portage import _encodings
from portage import _unicode_encode

class EbuildPhase(CompositeTask):

	__slots__ = ("actionmap", "fd_pipes", "phase", "settings") + \
		("_ebuild_lock",)

	# FEATURES displayed prior to setup phase
	_features_display = (
		"ccache", "compressdebug", "distcc", "fakeroot",
		"installsources", "keeptemp", "keepwork", "network-sandbox",
		"network-sandbox-proxy", "nostrip", "preserve-libs", "sandbox",
		"selinux", "sesandbox", "splitdebug", "suidctl", "test",
		"userpriv", "usersandbox"
	)

	# Locked phases
	_locked_phases = ("setup", "preinst", "postinst", "prerm", "postrm")

	def _start(self):
		self.scheduler.run_until_complete(self._async_start())

	@coroutine
	def _async_start(self):

		need_builddir = self.phase not in EbuildProcess._phases_without_builddir

		if need_builddir:
			phase_completed_file = os.path.join(
				self.settings['PORTAGE_BUILDDIR'],
				".%sed" % self.phase.rstrip('e'))
			if not os.path.exists(phase_completed_file):
				# If the phase is really going to run then we want
				# to eliminate any stale elog messages that may
				# exist from a previous run.
				try:
					os.unlink(os.path.join(self.settings['T'],
						'logging', self.phase))
				except OSError:
					pass

		if self.phase in ('nofetch', 'pretend', 'setup'):

			use = self.settings.get('PORTAGE_BUILT_USE')
			if use is None:
				use = self.settings['PORTAGE_USE']

			maint_str = ""
			upstr_str = ""
			metadata_xml_path = os.path.join(os.path.dirname(self.settings['EBUILD']), "metadata.xml")
			if MetaDataXML is not None and os.path.isfile(metadata_xml_path):
				herds_path = os.path.join(self.settings['PORTDIR'],
					'metadata/herds.xml')
				try:
					metadata_xml = MetaDataXML(metadata_xml_path, herds_path)
					maint_str = metadata_xml.format_maintainer_string()
					upstr_str = metadata_xml.format_upstream_string()
				except SyntaxError:
					maint_str = "<invalid metadata.xml>"

			msg = []
			msg.append("Package:    %s" % self.settings.mycpv)
			if self.settings.get('PORTAGE_REPO_NAME'):
				msg.append("Repository: %s" % self.settings['PORTAGE_REPO_NAME'])
			if maint_str:
				msg.append("Maintainer: %s" % maint_str)
			if upstr_str:
				msg.append("Upstream:   %s" % upstr_str)

			msg.append("USE:        %s" % use)
			relevant_features = []
			enabled_features = self.settings.features
			for x in self._features_display:
				if x in enabled_features:
					relevant_features.append(x)
			if relevant_features:
				msg.append("FEATURES:   %s" % " ".join(relevant_features))

			# Force background=True for this header since it's intended
			# for the log and it doesn't necessarily need to be visible
			# elsewhere.
			yield self._elog('einfo', msg, background=True)

		if self.phase == 'package':
			if 'PORTAGE_BINPKG_TMPFILE' not in self.settings:
				self.settings['PORTAGE_BINPKG_TMPFILE'] = \
					os.path.join(self.settings['PKGDIR'],
					self.settings['CATEGORY'], self.settings['PF']) + '.tbz2'

		if self.phase in ("pretend", "prerm"):
			env_extractor = BinpkgEnvExtractor(background=self.background,
				scheduler=self.scheduler, settings=self.settings)
			if env_extractor.saved_env_exists():
				self._current_task = env_extractor
				yield env_extractor.async_start()
				yield env_extractor.async_wait()
				if self._default_exit(env_extractor) != os.EX_OK:
					self._async_wait()
					coroutine_return()
			# If the environment.bz2 doesn't exist, then ebuild.sh will
			# source the ebuild as a fallback.

		self._start_lock()

	def _start_lock(self):
		if (self.phase in self._locked_phases and
			"ebuild-locks" in self.settings.features):
			eroot = self.settings["EROOT"]
			lock_path = os.path.join(eroot, portage.VDB_PATH + "-ebuild")
			if os.access(os.path.dirname(lock_path), os.W_OK):
				self._ebuild_lock = AsynchronousLock(path=lock_path,
					scheduler=self.scheduler)
				self._start_task(self._ebuild_lock, self._lock_exit)
				return

		self._start_ebuild()

	def _lock_exit(self, ebuild_lock):
		if self._default_exit(ebuild_lock) != os.EX_OK:
			self.wait()
			return
		self._start_ebuild()

	def _get_log_path(self):
		# Don't open the log file during the clean phase since the
		# open file can result in an nfs lock on $T/build.log which
		# prevents the clean phase from removing $T.
		logfile = None
		if self.phase not in ("clean", "cleanrm") and \
			self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
			logfile = self.settings.get("PORTAGE_LOG_FILE")
		return logfile

	def _start_ebuild(self):
		if self.phase == "package":
			self._start_task(PackagePhase(actionmap=self.actionmap,
				background=self.background, fd_pipes=self.fd_pipes,
				logfile=self._get_log_path(), scheduler=self.scheduler,
				settings=self.settings), self._ebuild_exit)
			return

		if self.phase == "unpack":
			alist = self.settings.configdict["pkg"].get("A", "").split()
			_prepare_fake_distdir(self.settings, alist)
			_prepare_fake_filesdir(self.settings)

		fd_pipes = self.fd_pipes
		if fd_pipes is None:
			if not self.background and self.phase == 'nofetch':
				# All the pkg_nofetch output goes to stderr since
				# it's considered to be an error message.
				fd_pipes = {1 : sys.__stderr__.fileno()}

		ebuild_process = EbuildProcess(actionmap=self.actionmap,
			background=self.background, fd_pipes=fd_pipes,
			logfile=self._get_log_path(), phase=self.phase,
			scheduler=self.scheduler, settings=self.settings)

		self._start_task(ebuild_process, self._ebuild_exit)

	def _ebuild_exit(self, ebuild_process):
		self._assert_current(ebuild_process)
		if self._ebuild_lock is None:
			self._ebuild_exit_unlocked(ebuild_process)
		else:
			self._start_task(
				AsyncTaskFuture(future=self._ebuild_lock.async_unlock()),
				functools.partial(self._ebuild_exit_unlocked, ebuild_process))

	def _ebuild_exit_unlocked(self, ebuild_process, unlock_task=None):
		if unlock_task is not None:
			self._assert_current(unlock_task)
			if unlock_task.cancelled:
				self._default_final_exit(unlock_task)
				return

			# Normally, async_unlock should not raise an exception here.
			unlock_task.future.result()

		fail = False
		if ebuild_process.returncode != os.EX_OK:
			self.returncode = ebuild_process.returncode
			if self.phase == "test" and \
				"test-fail-continue" in self.settings.features:
				# mark test phase as complete (bug #452030)
				try:
					open(_unicode_encode(os.path.join(
						self.settings["PORTAGE_BUILDDIR"], ".tested"),
						encoding=_encodings['fs'], errors='strict'),
						'wb').close()
				except OSError:
					pass
			else:
				fail = True

		if not fail:
			self.returncode = None

		logfile = self._get_log_path()

		if self.phase == "install":
			out = io.StringIO()
			_check_build_log(self.settings, out=out)
			msg = out.getvalue()
			self.scheduler.output(msg, log_path=logfile)

		if fail:
			self._die_hooks()
			return

		settings = self.settings
		_post_phase_userpriv_perms(settings)

		if self.phase == "unpack":
			# Bump WORKDIR timestamp, in case tar gave it a timestamp
			# that will interfere with distfiles / WORKDIR timestamp
			# comparisons as reported in bug #332217. Also, fix
			# ownership since tar can change that too.
			os.utime(settings["WORKDIR"], None)
			_prepare_workdir(settings)
		elif self.phase == "install":
			out = io.StringIO()
			_post_src_install_write_metadata(settings)
			_post_src_install_uid_fix(settings, out)
			msg = out.getvalue()
			if msg:
				self.scheduler.output(msg, log_path=logfile)
		elif self.phase == "preinst":
			_preinst_bsdflags(settings)
		elif self.phase == "postinst":
			_postinst_bsdflags(settings)

		post_phase_cmds = _post_phase_cmds.get(self.phase)
		if post_phase_cmds is not None:
			if logfile is not None and self.phase in ("install",):
				# Log to a temporary file, since the code we are running
				# reads PORTAGE_LOG_FILE for QA checks, and we want to
				# avoid annoying "gzip: unexpected end of file" messages
				# when FEATURES=compress-build-logs is enabled.
				fd, logfile = tempfile.mkstemp()
				os.close(fd)
			post_phase = _PostPhaseCommands(background=self.background,
				commands=post_phase_cmds, elog=self._elog, fd_pipes=self.fd_pipes,
				logfile=logfile, phase=self.phase, scheduler=self.scheduler,
				settings=settings)
			self._start_task(post_phase, self._post_phase_exit)
			return

		# this point is not reachable if there was a failure and
		# we returned for die_hooks above, so returncode must
		# indicate success (especially if ebuild_process.returncode
		# is unsuccessful and test-fail-continue came into play)
		self.returncode = os.EX_OK
		self._current_task = None
		self.wait()

	def _post_phase_exit(self, post_phase):

		self._assert_current(post_phase)

		log_path = None
		if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
			log_path = self.settings.get("PORTAGE_LOG_FILE")

		if post_phase.logfile is not None and \
			post_phase.logfile != log_path:
			# We were logging to a temp file (see above), so append
			# temp file to main log and remove temp file.
			self._append_temp_log(post_phase.logfile, log_path)

		if self._final_exit(post_phase) != os.EX_OK:
			writemsg("!!! post %s failed; exiting.\n" % self.phase,
				noiselevel=-1)
			self._die_hooks()
			return

		self._current_task = None
		self.wait()
		return

	def _append_temp_log(self, temp_log, log_path):

		temp_file = open(_unicode_encode(temp_log,
			encoding=_encodings['fs'], errors='strict'), 'rb')

		log_file, log_file_real = self._open_log(log_path)

		for line in temp_file:
			log_file.write(line)

		temp_file.close()
		log_file.close()
		if log_file_real is not log_file:
			log_file_real.close()
		os.unlink(temp_log)

	def _open_log(self, log_path):

		f = open(_unicode_encode(log_path,
			encoding=_encodings['fs'], errors='strict'),
			mode='ab')
		f_real = f

		if log_path.endswith('.gz'):
			f =  gzip.GzipFile(filename='', mode='ab', fileobj=f)

		return (f, f_real)

	def _die_hooks(self):
		self.returncode = None
		phase = 'die_hooks'
		die_hooks = MiscFunctionsProcess(background=self.background,
			commands=[phase], phase=phase, logfile=self._get_log_path(),
			fd_pipes=self.fd_pipes, scheduler=self.scheduler,
			settings=self.settings)
		self._start_task(die_hooks, self._die_hooks_exit)

	def _die_hooks_exit(self, die_hooks):
		if self.phase != 'clean' and \
			'noclean' not in self.settings.features and \
			'fail-clean' in self.settings.features:
			self._default_exit(die_hooks)
			self._fail_clean()
			return
		self._final_exit(die_hooks)
		self.returncode = 1
		self.wait()

	def _fail_clean(self):
		self.returncode = None
		portage.elog.elog_process(self.settings.mycpv, self.settings)
		phase = "clean"
		clean_phase = EbuildPhase(background=self.background,
			fd_pipes=self.fd_pipes, phase=phase, scheduler=self.scheduler,
			settings=self.settings)
		self._start_task(clean_phase, self._fail_clean_exit)
		return

	def _fail_clean_exit(self, clean_phase):
		self._final_exit(clean_phase)
		self.returncode = 1
		self.wait()

	@coroutine
	def _elog(self, elog_funcname, lines, background=None):
		if background is None:
			background = self.background
		out = io.StringIO()
		phase = self.phase
		elog_func = getattr(elog_messages, elog_funcname)
		global_havecolor = portage.output.havecolor
		try:
			portage.output.havecolor = \
				self.settings.get('NOCOLOR', 'false').lower() in ('no', 'false')
			for line in lines:
				elog_func(line, phase=phase, key=self.settings.mycpv, out=out)
		finally:
			portage.output.havecolor = global_havecolor
		msg = out.getvalue()
		if msg:
			build_logger = None
			try:
				log_file = None
				log_path = None
				if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
					log_path = self.settings.get("PORTAGE_LOG_FILE")
				if log_path:
					build_logger = BuildLogger(env=self.settings.environ(),
						log_path=log_path,
						log_filter_file=self.settings.get('PORTAGE_LOG_FILTER_FILE_CMD'),
						scheduler=self.scheduler)
					yield build_logger.async_start()
					log_file = build_logger.stdin

				yield self.scheduler.async_output(msg, log_file=log_file,
					background=background)

				if build_logger is not None:
					build_logger.stdin.close()
					yield build_logger.async_wait()
			except asyncio.CancelledError:
				if build_logger is not None:
					build_logger.cancel()
				raise


class _PostPhaseCommands(CompositeTask):

	__slots__ = ("commands", "elog", "fd_pipes", "logfile", "phase", "settings")

	def _start(self):
		if isinstance(self.commands, list):
			cmds = [({}, self.commands)]
		else:
			cmds = list(self.commands)

		if 'selinux' not in self.settings.features:
			cmds = [(kwargs, commands) for kwargs, commands in
				cmds if not kwargs.get('selinux_only')]

		tasks = TaskSequence()
		for kwargs, commands in cmds:
			# Select args intended for MiscFunctionsProcess.
			kwargs = dict((k, v) for k, v in kwargs.items()
				if k in ('ld_preload_sandbox',))
			tasks.add(MiscFunctionsProcess(background=self.background,
				commands=commands, fd_pipes=self.fd_pipes,
				logfile=self.logfile, phase=self.phase,
				scheduler=self.scheduler, settings=self.settings, **kwargs))

		self._start_task(tasks, self._commands_exit)

	def _commands_exit(self, task):

		if self._default_exit(task) != os.EX_OK:
			self._async_wait()
			return

		if self.phase == 'install':
			out = io.StringIO()
			_post_src_install_soname_symlinks(self.settings, out)
			msg = out.getvalue()
			if msg:
				self.scheduler.output(msg, log_path=self.settings.get("PORTAGE_LOG_FILE"))

			if 'qa-unresolved-soname-deps' in self.settings.features:
				# This operates on REQUIRES metadata generated by the above function call.
				future = self._soname_deps_qa()
				# If an unexpected exception occurs, then this will raise it.
				future.add_done_callback(lambda future: future.cancelled() or future.result())
				self._start_task(AsyncTaskFuture(future=future), self._default_final_exit)
			else:
				self._default_final_exit(task)
		else:
			self._default_final_exit(task)

	@coroutine
	def _soname_deps_qa(self):

		vardb = QueryCommand.get_db()[self.settings['EROOT']]['vartree'].dbapi

		all_provides = (yield self.scheduler.run_in_executor(ForkExecutor(loop=self.scheduler), _get_all_provides, vardb))

		unresolved = _get_unresolved_soname_deps(os.path.join(self.settings['PORTAGE_BUILDDIR'], 'build-info'), all_provides)

		if unresolved:
			unresolved.sort()
			qa_msg = ["QA Notice: Unresolved soname dependencies:"]
			qa_msg.append("")
			qa_msg.extend("\t%s: %s" % (filename, " ".join(sorted(soname_deps)))
				for filename, soname_deps in unresolved)
			qa_msg.append("")
			yield self.elog("eqawarn", qa_msg)