Support PORTAGE_LOG_FILTER_FILE_CMD (bug 709746)

This variable specifies a command that filters build log output to a
log file. The plan is to extend this to support a separate filter for
tty output in the future.

In order to enable the EbuildPhase class to write elog messages to
the build log with PORTAGE_LOG_FILTER_FILE_CMD support, convert its
_elog method to a coroutine, and add a SchedulerInterface async_output
method for it to use.

Use a new BuildLogger class to manage log output (with or without a
filter command), with compression support provided by PipeLogger.
BuildLogger has a stdin property which provides access to a writable
binary file stream (refers to a pipe) that log content is written to.

Bug: https://bugs.gentoo.org/709746
Reviewed-by: Brian Dolbec <dolsen@gentoo.org>
Signed-off-by: Zac Medico <zmedico@gentoo.org>

1 files changed, 44 insertions, 14 deletions

diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py
index 395d66bb9..f96911571 100644
--- a/lib/_emerge/SpawnProcess.py
+++ b/lib/_emerge/SpawnProcess.py
@@ -1,4 +1,4 @@
-# Copyright 2008-2018 Gentoo Foundation
+# Copyright 2008-2020 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 try:
@@ -19,7 +19,10 @@ from portage.const import BASH_BINARY
 from portage.localization import _
 from portage.output import EOutput
 from portage.util import writemsg_level
+from portage.util._async.BuildLogger import BuildLogger
 from portage.util._async.PipeLogger import PipeLogger
+from portage.util.futures import asyncio
+from portage.util.futures.compat_coroutine import coroutine
 
 class SpawnProcess(SubProcess):
 
@@ -34,8 +37,8 @@ class SpawnProcess(SubProcess):
 		"path_lookup", "pre_exec", "close_fds", "cgroup",
 		"unshare_ipc", "unshare_mount", "unshare_pid", "unshare_net")
 
-	__slots__ = ("args",) + \
-		_spawn_kwarg_names + ("_pipe_logger", "_selinux_type",)
+	__slots__ = ("args", "log_filter_file") + \
+		_spawn_kwarg_names + ("_main_task", "_selinux_type",)
 
 	# Max number of attempts to kill the processes listed in cgroup.procs,
 	# given that processes may fork before they can be killed.
@@ -137,13 +140,43 @@ class SpawnProcess(SubProcess):
 						fcntl.fcntl(stdout_fd,
 						fcntl.F_GETFD) | fcntl.FD_CLOEXEC)
 
-		self._pipe_logger = PipeLogger(background=self.background,
+		build_logger = BuildLogger(env=self.env,
+			log_path=log_file_path,
+			log_filter_file=self.log_filter_file,
+			scheduler=self.scheduler)
+		build_logger.start()
+
+		pipe_logger = PipeLogger(background=self.background,
 			scheduler=self.scheduler, input_fd=master_fd,
-			log_file_path=log_file_path,
+			log_file_path=build_logger.stdin,
 			stdout_fd=stdout_fd)
-		self._pipe_logger.addExitListener(self._pipe_logger_exit)
-		self._pipe_logger.start()
+
+		pipe_logger.start()
+
 		self._registered = True
+		self._main_task = asyncio.ensure_future(self._main(build_logger, pipe_logger), loop=self.scheduler)
+		self._main_task.add_done_callback(self._main_exit)
+
+	@coroutine
+	def _main(self, build_logger, pipe_logger):
+		try:
+			if pipe_logger.poll() is None:
+				yield pipe_logger.async_wait()
+			if build_logger.poll() is None:
+				yield build_logger.async_wait()
+		except asyncio.CancelledError:
+			if pipe_logger.poll() is None:
+				pipe_logger.cancel()
+			if build_logger.poll() is None:
+				build_logger.cancel()
+			raise
+
+	def _main_exit(self, main_task):
+		try:
+			main_task.result()
+		except asyncio.CancelledError:
+			self.cancel()
+		self._async_waitpid()
 
 	def _can_log(self, slave_fd):
 		return True
@@ -167,20 +200,17 @@ class SpawnProcess(SubProcess):
 
 		return spawn_func(args, **kwargs)
 
-	def _pipe_logger_exit(self, pipe_logger):
-		self._pipe_logger = None
-		self._async_waitpid()
-
 	def _unregister(self):
 		SubProcess._unregister(self)
 		if self.cgroup is not None:
 			self._cgroup_cleanup()
 			self.cgroup = None
-		if self._pipe_logger is not None:
-			self._pipe_logger.cancel()
-			self._pipe_logger = None
+		if self._main_task is not None:
+			self._main_task.done() or self._main_task.cancel()
 
 	def _cancel(self):
+		if self._main_task is not None:
+			self._main_task.done() or self._main_task.cancel()
 		SubProcess._cancel(self)
 		self._cgroup_cleanup()