diff options
author | Zac Medico <zmedico@gentoo.org> | 2020-02-29 18:17:52 -0800 |
---|---|---|
committer | Zac Medico <zmedico@gentoo.org> | 2020-02-29 20:29:41 -0800 |
commit | 73f72f526a66b9953a46868cc1390fde2820997f (patch) | |
tree | f6463a8896a80e667d0782fc47de312f12a2a75c /lib/portage | |
parent | _PostPhaseCommands: avoid CancelledError in _soname_deps_qa done callback (diff) | |
download | portage-73f72f526a66b9953a46868cc1390fde2820997f.tar.gz portage-73f72f526a66b9953a46868cc1390fde2820997f.tar.bz2 portage-73f72f526a66b9953a46868cc1390fde2820997f.zip |
Support PORTAGE_LOG_FILTER_FILE (bug 709746)
This variable specifies a command that filters build log output to a
log file. The plan is to extend this to support a separate filter for
tty output in the future.
In order to enable the EbuildPhase class to write elog messages to
the build log with PORTAGE_LOG_FILTER_FILE support, convert its _elog
method to a coroutine, and add a SchedulerInterface async_output
method for it to use.
Bug: https://bugs.gentoo.org/709746
Signed-off-by: Zac Medico <zmedico@gentoo.org>
Diffstat (limited to 'lib/portage')
-rw-r--r-- | lib/portage/package/ebuild/_config/special_env_vars.py | 8 | ||||
-rw-r--r-- | lib/portage/util/_async/BuildLogger.py | 116 | ||||
-rw-r--r-- | lib/portage/util/_async/SchedulerInterface.py | 32 |
3 files changed, 152 insertions, 4 deletions
diff --git a/lib/portage/package/ebuild/_config/special_env_vars.py b/lib/portage/package/ebuild/_config/special_env_vars.py index dc01339f7..dd8105123 100644 --- a/lib/portage/package/ebuild/_config/special_env_vars.py +++ b/lib/portage/package/ebuild/_config/special_env_vars.py @@ -1,4 +1,4 @@ -# Copyright 2010-2019 Gentoo Authors +# Copyright 2010-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from __future__ import unicode_literals @@ -175,7 +175,7 @@ environ_filter += [ "PORTAGE_RO_DISTDIRS", "PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS", "PORTAGE_RSYNC_RETRIES", "PORTAGE_SSH_OPTS", "PORTAGE_SYNC_STALE", - "PORTAGE_USE", + "PORTAGE_USE", "PORTAGE_LOG_FILTER_FILE", "PORTAGE_LOGDIR", "PORTAGE_LOGDIR_CLEAN", "QUICKPKG_DEFAULT_OPTS", "REPOMAN_DEFAULT_OPTS", "RESUMECOMMAND", "RESUMECOMMAND_FTP", @@ -204,7 +204,9 @@ default_globals = { 'PORTAGE_BZIP2_COMMAND': 'bzip2', } -validate_commands = ('PORTAGE_BZIP2_COMMAND', 'PORTAGE_BUNZIP2_COMMAND',) +validate_commands = ('PORTAGE_BZIP2_COMMAND', 'PORTAGE_BUNZIP2_COMMAND', + 'PORTAGE_LOG_FILTER_FILE', +) # To enhance usability, make some vars case insensitive # by forcing them to lower case. diff --git a/lib/portage/util/_async/BuildLogger.py b/lib/portage/util/_async/BuildLogger.py new file mode 100644 index 000000000..4873d9750 --- /dev/null +++ b/lib/portage/util/_async/BuildLogger.py @@ -0,0 +1,116 @@ +# Copyright 2020 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +from portage import os +from portage.util import shlex_split +from _emerge.AsynchronousTask import AsynchronousTask +from portage.util._async.PipeLogger import PipeLogger +from portage.util.futures import asyncio +from portage.util.futures.compat_coroutine import coroutine + + +class BuildLogger(AsynchronousTask): + """ + Write to a log file, with compression support provided by PipeLogger. + If the log_filter_file parameter is specified, then it is interpreted + as a command to execute which filters log output (see the + PORTAGE_LOG_FILTER_FILE variable in make.conf(5)). The stdin property + provides access to a writable binary file stream (refers to a pipe) + that log content should be written to (usually redirected from + subprocess stdout and stderr streams). + """ + + __slots__ = ('env', 'log_path', 'log_filter_file', '_main_task', '_stdin') + + @property + def stdin(self): + return self._stdin + + def _start(self): + self.scheduler.run_until_complete(self._async_start()) + + @coroutine + def _async_start(self): + pipe_logger = None + filter_proc = None + try: + log_input = None + if self.log_path is not None: + log_filter_file = self.log_filter_file + if log_filter_file is not None: + split_value = shlex_split(log_filter_file) + log_filter_file = split_value if split_value else None + if log_filter_file: + filter_input, stdin = os.pipe() + log_input, filter_output = os.pipe() + try: + filter_proc = yield asyncio.create_subprocess_exec( + *log_filter_file, + env=self.env, + stdin=filter_input, + stdout=filter_output, + stderr=filter_output, + loop=self.scheduler) + except EnvironmentError: + # Maybe the command is missing or broken somehow... + os.close(filter_input) + os.close(stdin) + os.close(log_input) + os.close(filter_output) + else: + self._stdin = os.fdopen(stdin, 'wb', 0) + os.close(filter_input) + os.close(filter_output) + + if self._stdin is None: + # Since log_filter_file is unspecified or refers to a file + # that is missing or broken somehow, create a pipe that + # logs directly to pipe_logger. + log_input, stdin = os.pipe() + self._stdin = os.fdopen(stdin, 'wb', 0) + + # Set background=True so that pipe_logger does not log to stdout. + pipe_logger = PipeLogger(background=True, + scheduler=self.scheduler, input_fd=log_input, + log_file_path=self.log_path) + + yield pipe_logger.async_start() + except asyncio.CancelledError: + if pipe_logger is not None and pipe_logger.poll() is None: + pipe_logger.cancel() + if filter_proc is not None and filter_proc.returncode is None: + filter_proc.terminate() + raise + + self._main_task = asyncio.ensure_future( + self._main(pipe_logger, filter_proc=filter_proc), loop=self.scheduler) + self._main_task.add_done_callback(self._main_exit) + + def _cancel(self): + if self._main_task is not None: + self._main_task.done() or self._main_task.cancel() + if self._stdin is not None and not self._stdin.closed: + self._stdin.close() + + @coroutine + def _main(self, pipe_logger, filter_proc=None): + try: + if pipe_logger.poll() is None: + yield pipe_logger.async_wait() + if filter_proc is not None and filter_proc.returncode is None: + yield filter_proc.wait() + except asyncio.CancelledError: + if pipe_logger.poll() is None: + pipe_logger.cancel() + if filter_proc is not None and filter_proc.returncode is None: + filter_proc.terminate() + raise + + def _main_exit(self, main_task): + try: + main_task.result() + except asyncio.CancelledError: + self.cancel() + self._was_cancelled() + self.returncode = self.returncode or 0 + self._async_wait() diff --git a/lib/portage/util/_async/SchedulerInterface.py b/lib/portage/util/_async/SchedulerInterface.py index ec6417da1..3ff250d1d 100644 --- a/lib/portage/util/_async/SchedulerInterface.py +++ b/lib/portage/util/_async/SchedulerInterface.py @@ -1,4 +1,4 @@ -# Copyright 2012-2018 Gentoo Foundation +# Copyright 2012-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import gzip @@ -7,6 +7,8 @@ import errno from portage import _encodings from portage import _unicode_encode from portage.util import writemsg_level +from portage.util.futures._asyncio.streams import _writer +from portage.util.futures.compat_coroutine import coroutine from ..SlotObject import SlotObject class SchedulerInterface(SlotObject): @@ -53,6 +55,34 @@ class SchedulerInterface(SlotObject): def _return_false(): return False + @coroutine + def async_output(self, msg, log_file=None, background=None, + level=0, noiselevel=-1): + """ + Output a msg to stdio (if not in background) and to a log file + if provided. + + @param msg: a message string, including newline if appropriate + @type msg: str + @param log_file: log file in binary mode + @type log_file: file + @param background: send messages only to log (not to stdio) + @type background: bool + @param level: a numeric logging level (see the logging module) + @type level: int + @param noiselevel: passed directly to writemsg + @type noiselevel: int + """ + global_background = self._is_background() + if background is None or global_background: + background = global_background + + if not background: + writemsg_level(msg, level=level, noiselevel=noiselevel) + + if log_file is not None: + yield _writer(log_file, _unicode_encode(msg)) + def output(self, msg, log_path=None, background=None, level=0, noiselevel=-1): """ |