aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorZac Medico <zmedico@gentoo.org>2020-02-29 18:17:52 -0800
committerZac Medico <zmedico@gentoo.org>2020-06-22 19:13:06 -0700
commitdd69ce742c62b9515cf7ae37e46bcf7f178777db (patch)
treecac54966c47ff2972955cbcf4c3f4a384c49c7dc /lib/_emerge
parentPipeLogger: non-blocking write to pipe (bug 709746) (diff)
downloadportage-dd69ce742c62b9515cf7ae37e46bcf7f178777db.tar.gz
portage-dd69ce742c62b9515cf7ae37e46bcf7f178777db.tar.bz2
portage-dd69ce742c62b9515cf7ae37e46bcf7f178777db.zip
Support PORTAGE_LOG_FILTER_FILE_CMD (bug 709746)
This variable specifies a command that filters build log output to a log file. The plan is to extend this to support a separate filter for tty output in the future. In order to enable the EbuildPhase class to write elog messages to the build log with PORTAGE_LOG_FILTER_FILE_CMD support, convert its _elog method to a coroutine, and add a SchedulerInterface async_output method for it to use. Use a new BuildLogger class to manage log output (with or without a filter command), with compression support provided by PipeLogger. BuildLogger has a stdin property which provides access to a writable binary file stream (refers to a pipe) that log content is written to. Bug: https://bugs.gentoo.org/709746 Reviewed-by: Brian Dolbec <dolsen@gentoo.org> Signed-off-by: Zac Medico <zmedico@gentoo.org>
Diffstat (limited to 'lib/_emerge')
-rw-r--r--lib/_emerge/AbstractEbuildProcess.py3
-rw-r--r--lib/_emerge/BinpkgFetcher.py3
-rw-r--r--lib/_emerge/EbuildFetcher.py3
-rw-r--r--lib/_emerge/EbuildPhase.py47
-rw-r--r--lib/_emerge/SpawnProcess.py58
5 files changed, 90 insertions, 24 deletions
diff --git a/lib/_emerge/AbstractEbuildProcess.py b/lib/_emerge/AbstractEbuildProcess.py
index 1c1955cfe..ae1aae55f 100644
--- a/lib/_emerge/AbstractEbuildProcess.py
+++ b/lib/_emerge/AbstractEbuildProcess.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2019 Gentoo Foundation
+# Copyright 1999-2020 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import errno
@@ -196,6 +196,7 @@ class AbstractEbuildProcess(SpawnProcess):
null_fd = os.open('/dev/null', os.O_RDONLY)
self.fd_pipes[0] = null_fd
+ self.log_filter_file = self.settings.get('PORTAGE_LOG_FILTER_FILE_CMD')
try:
SpawnProcess._start(self)
finally:
diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py
index 36d027de3..2e5861cc1 100644
--- a/lib/_emerge/BinpkgFetcher.py
+++ b/lib/_emerge/BinpkgFetcher.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2018 Gentoo Foundation
+# Copyright 1999-2020 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import functools
@@ -158,6 +158,7 @@ class _BinpkgFetcherProcess(SpawnProcess):
self.env = fetch_env
if settings.selinux_enabled():
self._selinux_type = settings["PORTAGE_FETCH_T"]
+ self.log_filter_file = settings.get('PORTAGE_LOG_FILTER_FILE_CMD')
SpawnProcess._start(self)
def _pipe(self, fd_pipes):
diff --git a/lib/_emerge/EbuildFetcher.py b/lib/_emerge/EbuildFetcher.py
index 1e40994fb..55349c33c 100644
--- a/lib/_emerge/EbuildFetcher.py
+++ b/lib/_emerge/EbuildFetcher.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2018 Gentoo Foundation
+# Copyright 1999-2020 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import copy
@@ -225,6 +225,7 @@ class _EbuildFetcherProcess(ForkProcess):
settings["NOCOLOR"] = nocolor
self._settings = settings
+ self.log_filter_file = settings.get('PORTAGE_LOG_FILTER_FILE_CMD')
ForkProcess._start(self)
# Free settings now since it's no longer needed in
diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index 477e0ba97..ddb3dc719 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -26,6 +26,8 @@ from portage.package.ebuild.prepare_build_dirs import (_prepare_workdir,
from portage.util.futures.compat_coroutine import coroutine
from portage.util import writemsg
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
+from portage.util._async.BuildLogger import BuildLogger
+from portage.util.futures import asyncio
from portage.util.futures.executor.fork import ForkExecutor
try:
@@ -69,6 +71,11 @@ class EbuildPhase(CompositeTask):
_locked_phases = ("setup", "preinst", "postinst", "prerm", "postrm")
def _start(self):
+ future = asyncio.ensure_future(self._async_start(), loop=self.scheduler)
+ self._start_task(AsyncTaskFuture(future=future), self._async_start_exit)
+
+ @coroutine
+ def _async_start(self):
need_builddir = self.phase not in EbuildProcess._phases_without_builddir
@@ -126,7 +133,7 @@ class EbuildPhase(CompositeTask):
# Force background=True for this header since it's intended
# for the log and it doesn't necessarily need to be visible
# elsewhere.
- self._elog('einfo', msg, background=True)
+ yield self._elog('einfo', msg, background=True)
if self.phase == 'package':
if 'PORTAGE_BINPKG_TMPFILE' not in self.settings:
@@ -134,6 +141,12 @@ class EbuildPhase(CompositeTask):
os.path.join(self.settings['PKGDIR'],
self.settings['CATEGORY'], self.settings['PF']) + '.tbz2'
+ def _async_start_exit(self, task):
+ task.future.cancelled() or task.future.result()
+ if self._default_exit(task) != os.EX_OK:
+ self.wait()
+ return
+
if self.phase in ("pretend", "prerm"):
env_extractor = BinpkgEnvExtractor(background=self.background,
scheduler=self.scheduler, settings=self.settings)
@@ -391,6 +404,7 @@ class EbuildPhase(CompositeTask):
self.returncode = 1
self.wait()
+ @coroutine
def _elog(self, elog_funcname, lines, background=None):
if background is None:
background = self.background
@@ -407,11 +421,30 @@ class EbuildPhase(CompositeTask):
portage.output.havecolor = global_havecolor
msg = out.getvalue()
if msg:
- log_path = None
- if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
- log_path = self.settings.get("PORTAGE_LOG_FILE")
- self.scheduler.output(msg, log_path=log_path,
- background=background)
+ build_logger = None
+ try:
+ log_file = None
+ log_path = None
+ if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
+ log_path = self.settings.get("PORTAGE_LOG_FILE")
+ if log_path:
+ build_logger = BuildLogger(env=self.settings.environ(),
+ log_path=log_path,
+ log_filter_file=self.settings.get('PORTAGE_LOG_FILTER_FILE_CMD'),
+ scheduler=self.scheduler)
+ build_logger.start()
+ log_file = build_logger.stdin
+
+ yield self.scheduler.async_output(msg, log_file=log_file,
+ background=background)
+
+ if build_logger is not None:
+ build_logger.stdin.close()
+ yield build_logger.async_wait()
+ except asyncio.CancelledError:
+ if build_logger is not None:
+ build_logger.cancel()
+ raise
class _PostPhaseCommands(CompositeTask):
@@ -480,4 +513,4 @@ class _PostPhaseCommands(CompositeTask):
qa_msg.extend("\t%s: %s" % (filename, " ".join(sorted(soname_deps)))
for filename, soname_deps in unresolved)
qa_msg.append("")
- self.elog("eqawarn", qa_msg)
+ yield self.elog("eqawarn", qa_msg)
diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py
index 395d66bb9..f96911571 100644
--- a/lib/_emerge/SpawnProcess.py
+++ b/lib/_emerge/SpawnProcess.py
@@ -1,4 +1,4 @@
-# Copyright 2008-2018 Gentoo Foundation
+# Copyright 2008-2020 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
try:
@@ -19,7 +19,10 @@ from portage.const import BASH_BINARY
from portage.localization import _
from portage.output import EOutput
from portage.util import writemsg_level
+from portage.util._async.BuildLogger import BuildLogger
from portage.util._async.PipeLogger import PipeLogger
+from portage.util.futures import asyncio
+from portage.util.futures.compat_coroutine import coroutine
class SpawnProcess(SubProcess):
@@ -34,8 +37,8 @@ class SpawnProcess(SubProcess):
"path_lookup", "pre_exec", "close_fds", "cgroup",
"unshare_ipc", "unshare_mount", "unshare_pid", "unshare_net")
- __slots__ = ("args",) + \
- _spawn_kwarg_names + ("_pipe_logger", "_selinux_type",)
+ __slots__ = ("args", "log_filter_file") + \
+ _spawn_kwarg_names + ("_main_task", "_selinux_type",)
# Max number of attempts to kill the processes listed in cgroup.procs,
# given that processes may fork before they can be killed.
@@ -137,13 +140,43 @@ class SpawnProcess(SubProcess):
fcntl.fcntl(stdout_fd,
fcntl.F_GETFD) | fcntl.FD_CLOEXEC)
- self._pipe_logger = PipeLogger(background=self.background,
+ build_logger = BuildLogger(env=self.env,
+ log_path=log_file_path,
+ log_filter_file=self.log_filter_file,
+ scheduler=self.scheduler)
+ build_logger.start()
+
+ pipe_logger = PipeLogger(background=self.background,
scheduler=self.scheduler, input_fd=master_fd,
- log_file_path=log_file_path,
+ log_file_path=build_logger.stdin,
stdout_fd=stdout_fd)
- self._pipe_logger.addExitListener(self._pipe_logger_exit)
- self._pipe_logger.start()
+
+ pipe_logger.start()
+
self._registered = True
+ self._main_task = asyncio.ensure_future(self._main(build_logger, pipe_logger), loop=self.scheduler)
+ self._main_task.add_done_callback(self._main_exit)
+
+ @coroutine
+ def _main(self, build_logger, pipe_logger):
+ try:
+ if pipe_logger.poll() is None:
+ yield pipe_logger.async_wait()
+ if build_logger.poll() is None:
+ yield build_logger.async_wait()
+ except asyncio.CancelledError:
+ if pipe_logger.poll() is None:
+ pipe_logger.cancel()
+ if build_logger.poll() is None:
+ build_logger.cancel()
+ raise
+
+ def _main_exit(self, main_task):
+ try:
+ main_task.result()
+ except asyncio.CancelledError:
+ self.cancel()
+ self._async_waitpid()
def _can_log(self, slave_fd):
return True
@@ -167,20 +200,17 @@ class SpawnProcess(SubProcess):
return spawn_func(args, **kwargs)
- def _pipe_logger_exit(self, pipe_logger):
- self._pipe_logger = None
- self._async_waitpid()
-
def _unregister(self):
SubProcess._unregister(self)
if self.cgroup is not None:
self._cgroup_cleanup()
self.cgroup = None
- if self._pipe_logger is not None:
- self._pipe_logger.cancel()
- self._pipe_logger = None
+ if self._main_task is not None:
+ self._main_task.done() or self._main_task.cancel()
def _cancel(self):
+ if self._main_task is not None:
+ self._main_task.done() or self._main_task.cancel()
SubProcess._cancel(self)
self._cgroup_cleanup()