aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorZac Medico <zmedico@gentoo.org>2020-02-29 18:17:52 -0800
committerZac Medico <zmedico@gentoo.org>2020-02-29 20:29:41 -0800
commit73f72f526a66b9953a46868cc1390fde2820997f (patch)
treef6463a8896a80e667d0782fc47de312f12a2a75c /lib/portage
parent_PostPhaseCommands: avoid CancelledError in _soname_deps_qa done callback (diff)
downloadportage-73f72f526a66b9953a46868cc1390fde2820997f.tar.gz
portage-73f72f526a66b9953a46868cc1390fde2820997f.tar.bz2
portage-73f72f526a66b9953a46868cc1390fde2820997f.zip
Support PORTAGE_LOG_FILTER_FILE (bug 709746)
This variable specifies a command that filters build log output to a log file. The plan is to extend this to support a separate filter for tty output in the future. In order to enable the EbuildPhase class to write elog messages to the build log with PORTAGE_LOG_FILTER_FILE support, convert its _elog method to a coroutine, and add a SchedulerInterface async_output method for it to use. Bug: https://bugs.gentoo.org/709746 Signed-off-by: Zac Medico <zmedico@gentoo.org>
Diffstat (limited to 'lib/portage')
-rw-r--r--lib/portage/package/ebuild/_config/special_env_vars.py8
-rw-r--r--lib/portage/util/_async/BuildLogger.py116
-rw-r--r--lib/portage/util/_async/SchedulerInterface.py32
3 files changed, 152 insertions, 4 deletions
diff --git a/lib/portage/package/ebuild/_config/special_env_vars.py b/lib/portage/package/ebuild/_config/special_env_vars.py
index dc01339f7..dd8105123 100644
--- a/lib/portage/package/ebuild/_config/special_env_vars.py
+++ b/lib/portage/package/ebuild/_config/special_env_vars.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2019 Gentoo Authors
+# Copyright 2010-2020 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from __future__ import unicode_literals
@@ -175,7 +175,7 @@ environ_filter += [
"PORTAGE_RO_DISTDIRS",
"PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS",
"PORTAGE_RSYNC_RETRIES", "PORTAGE_SSH_OPTS", "PORTAGE_SYNC_STALE",
- "PORTAGE_USE",
+ "PORTAGE_USE", "PORTAGE_LOG_FILTER_FILE",
"PORTAGE_LOGDIR", "PORTAGE_LOGDIR_CLEAN",
"QUICKPKG_DEFAULT_OPTS", "REPOMAN_DEFAULT_OPTS",
"RESUMECOMMAND", "RESUMECOMMAND_FTP",
@@ -204,7 +204,9 @@ default_globals = {
'PORTAGE_BZIP2_COMMAND': 'bzip2',
}
-validate_commands = ('PORTAGE_BZIP2_COMMAND', 'PORTAGE_BUNZIP2_COMMAND',)
+validate_commands = ('PORTAGE_BZIP2_COMMAND', 'PORTAGE_BUNZIP2_COMMAND',
+ 'PORTAGE_LOG_FILTER_FILE',
+)
# To enhance usability, make some vars case insensitive
# by forcing them to lower case.
diff --git a/lib/portage/util/_async/BuildLogger.py b/lib/portage/util/_async/BuildLogger.py
new file mode 100644
index 000000000..4873d9750
--- /dev/null
+++ b/lib/portage/util/_async/BuildLogger.py
@@ -0,0 +1,116 @@
+# Copyright 2020 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+from portage import os
+from portage.util import shlex_split
+from _emerge.AsynchronousTask import AsynchronousTask
+from portage.util._async.PipeLogger import PipeLogger
+from portage.util.futures import asyncio
+from portage.util.futures.compat_coroutine import coroutine
+
+
+class BuildLogger(AsynchronousTask):
+ """
+ Write to a log file, with compression support provided by PipeLogger.
+ If the log_filter_file parameter is specified, then it is interpreted
+ as a command to execute which filters log output (see the
+ PORTAGE_LOG_FILTER_FILE variable in make.conf(5)). The stdin property
+ provides access to a writable binary file stream (refers to a pipe)
+ that log content should be written to (usually redirected from
+ subprocess stdout and stderr streams).
+ """
+
+ __slots__ = ('env', 'log_path', 'log_filter_file', '_main_task', '_stdin')
+
+ @property
+ def stdin(self):
+ return self._stdin
+
+ def _start(self):
+ self.scheduler.run_until_complete(self._async_start())
+
+ @coroutine
+ def _async_start(self):
+ pipe_logger = None
+ filter_proc = None
+ try:
+ log_input = None
+ if self.log_path is not None:
+ log_filter_file = self.log_filter_file
+ if log_filter_file is not None:
+ split_value = shlex_split(log_filter_file)
+ log_filter_file = split_value if split_value else None
+ if log_filter_file:
+ filter_input, stdin = os.pipe()
+ log_input, filter_output = os.pipe()
+ try:
+ filter_proc = yield asyncio.create_subprocess_exec(
+ *log_filter_file,
+ env=self.env,
+ stdin=filter_input,
+ stdout=filter_output,
+ stderr=filter_output,
+ loop=self.scheduler)
+ except EnvironmentError:
+ # Maybe the command is missing or broken somehow...
+ os.close(filter_input)
+ os.close(stdin)
+ os.close(log_input)
+ os.close(filter_output)
+ else:
+ self._stdin = os.fdopen(stdin, 'wb', 0)
+ os.close(filter_input)
+ os.close(filter_output)
+
+ if self._stdin is None:
+ # Since log_filter_file is unspecified or refers to a file
+ # that is missing or broken somehow, create a pipe that
+ # logs directly to pipe_logger.
+ log_input, stdin = os.pipe()
+ self._stdin = os.fdopen(stdin, 'wb', 0)
+
+ # Set background=True so that pipe_logger does not log to stdout.
+ pipe_logger = PipeLogger(background=True,
+ scheduler=self.scheduler, input_fd=log_input,
+ log_file_path=self.log_path)
+
+ yield pipe_logger.async_start()
+ except asyncio.CancelledError:
+ if pipe_logger is not None and pipe_logger.poll() is None:
+ pipe_logger.cancel()
+ if filter_proc is not None and filter_proc.returncode is None:
+ filter_proc.terminate()
+ raise
+
+ self._main_task = asyncio.ensure_future(
+ self._main(pipe_logger, filter_proc=filter_proc), loop=self.scheduler)
+ self._main_task.add_done_callback(self._main_exit)
+
+ def _cancel(self):
+ if self._main_task is not None:
+ self._main_task.done() or self._main_task.cancel()
+ if self._stdin is not None and not self._stdin.closed:
+ self._stdin.close()
+
+ @coroutine
+ def _main(self, pipe_logger, filter_proc=None):
+ try:
+ if pipe_logger.poll() is None:
+ yield pipe_logger.async_wait()
+ if filter_proc is not None and filter_proc.returncode is None:
+ yield filter_proc.wait()
+ except asyncio.CancelledError:
+ if pipe_logger.poll() is None:
+ pipe_logger.cancel()
+ if filter_proc is not None and filter_proc.returncode is None:
+ filter_proc.terminate()
+ raise
+
+ def _main_exit(self, main_task):
+ try:
+ main_task.result()
+ except asyncio.CancelledError:
+ self.cancel()
+ self._was_cancelled()
+ self.returncode = self.returncode or 0
+ self._async_wait()
diff --git a/lib/portage/util/_async/SchedulerInterface.py b/lib/portage/util/_async/SchedulerInterface.py
index ec6417da1..3ff250d1d 100644
--- a/lib/portage/util/_async/SchedulerInterface.py
+++ b/lib/portage/util/_async/SchedulerInterface.py
@@ -1,4 +1,4 @@
-# Copyright 2012-2018 Gentoo Foundation
+# Copyright 2012-2020 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import gzip
@@ -7,6 +7,8 @@ import errno
from portage import _encodings
from portage import _unicode_encode
from portage.util import writemsg_level
+from portage.util.futures._asyncio.streams import _writer
+from portage.util.futures.compat_coroutine import coroutine
from ..SlotObject import SlotObject
class SchedulerInterface(SlotObject):
@@ -53,6 +55,34 @@ class SchedulerInterface(SlotObject):
def _return_false():
return False
+ @coroutine
+ def async_output(self, msg, log_file=None, background=None,
+ level=0, noiselevel=-1):
+ """
+ Output a msg to stdio (if not in background) and to a log file
+ if provided.
+
+ @param msg: a message string, including newline if appropriate
+ @type msg: str
+ @param log_file: log file in binary mode
+ @type log_file: file
+ @param background: send messages only to log (not to stdio)
+ @type background: bool
+ @param level: a numeric logging level (see the logging module)
+ @type level: int
+ @param noiselevel: passed directly to writemsg
+ @type noiselevel: int
+ """
+ global_background = self._is_background()
+ if background is None or global_background:
+ background = global_background
+
+ if not background:
+ writemsg_level(msg, level=level, noiselevel=noiselevel)
+
+ if log_file is not None:
+ yield _writer(log_file, _unicode_encode(msg))
+
def output(self, msg, log_path=None, background=None,
level=0, noiselevel=-1):
"""