aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorZac Medico <zmedico@gentoo.org>2019-11-24 21:08:14 -0800
committerZac Medico <zmedico@gentoo.org>2019-11-26 19:19:20 -0800
commit8faad11a18fcc33329931a75002f293e8fa462eb (patch)
tree3c21750fa5a368c7840228a25c85097645c98c7a /lib/_emerge
parentman/emerge.1: fix \fB typo (diff)
downloadportage-8faad11a18fcc33329931a75002f293e8fa462eb.tar.gz
portage-8faad11a18fcc33329931a75002f293e8fa462eb.tar.bz2
portage-8faad11a18fcc33329931a75002f293e8fa462eb.zip
emerge: add --quickpkg-direct option
Enable use of installed packages directly as binary packages. This is similar to using binary packages produced by quickpkg(1), but installed packages are used directly as though they are binary packages. This option only works in combination with the --root=DIR option, and it comes with the caveat that packages are only allowed to be installed into the root that is specified by the --root=DIR option. The other root which serves as a source of packages is assumed to be immutable during the entire operation (similar to --buildpkgonly mode). Default behavior for handling of protected configuration files is controlled by the QUICKPKG_DEFAULT_OPTS variable. When a configuration file is not included because it is protected, an ewarn message is logged. Suggested use cases: * Install packages from a buildtime container into an empty root, in order to create a minimal runtime container (which need not include a package manager). In a multi-stage Dockerfile, install runtime files to an empty directory in the build stage, and in the final stage use COPY to populate a container with the contents of that directory. For greater efficiency, use buildah to install directly into a mounted container, avoiding the COPY step. Use the emerge --usepkgonly and --ignore-soname-deps=n options to account for soname dependencies, allowing implicit system dependencies such as glibc to be automatically pulled into the runtime image. * Enable a live usb, iso, or pxe image to act as a binary installer that uses packages installed in the live image as a source of binary packages. Bug: https://bugs.gentoo.org/699986 Signed-off-by: Zac Medico <zmedico@gentoo.org>
Diffstat (limited to 'lib/_emerge')
-rw-r--r--lib/_emerge/Binpkg.py59
-rw-r--r--lib/_emerge/Scheduler.py7
-rw-r--r--lib/_emerge/actions.py37
-rw-r--r--lib/_emerge/depgraph.py19
-rw-r--r--lib/_emerge/main.py7
5 files changed, 97 insertions, 32 deletions
diff --git a/lib/_emerge/Binpkg.py b/lib/_emerge/Binpkg.py
index f9cffa26d..b5a69f8e7 100644
--- a/lib/_emerge/Binpkg.py
+++ b/lib/_emerge/Binpkg.py
@@ -7,7 +7,6 @@ import _emerge.emergelog
from _emerge.EbuildPhase import EbuildPhase
from _emerge.BinpkgFetcher import BinpkgFetcher
from _emerge.BinpkgEnvExtractor import BinpkgEnvExtractor
-from _emerge.BinpkgExtractorAsync import BinpkgExtractorAsync
from _emerge.CompositeTask import CompositeTask
from _emerge.BinpkgVerifier import BinpkgVerifier
from _emerge.EbuildMerge import EbuildMerge
@@ -16,6 +15,7 @@ from _emerge.SpawnProcess import SpawnProcess
from portage.eapi import eapi_exports_replace_vars
from portage.util import ensure_dirs
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
+from portage.util.futures.compat_coroutine import coroutine
import portage
from portage import os
from portage import shutil
@@ -135,11 +135,14 @@ class Binpkg(CompositeTask):
pkg = self.pkg
pkg_count = self.pkg_count
- fetcher = BinpkgFetcher(background=self.background,
- logfile=self.settings.get("PORTAGE_LOG_FILE"), pkg=self.pkg,
- pretend=self.opts.pretend, scheduler=self.scheduler)
+ fetcher = None
if self.opts.getbinpkg and self._bintree.isremote(pkg.cpv):
+
+ fetcher = BinpkgFetcher(background=self.background,
+ logfile=self.settings.get("PORTAGE_LOG_FILE"), pkg=self.pkg,
+ pretend=self.opts.pretend, scheduler=self.scheduler)
+
msg = " --- (%s of %s) Fetching Binary (%s::%s)" %\
(pkg_count.curval, pkg_count.maxval, pkg.cpv,
fetcher.pkg_path)
@@ -160,7 +163,7 @@ class Binpkg(CompositeTask):
# The fetcher only has a returncode when
# --getbinpkg is enabled.
- if fetcher.returncode is not None:
+ if fetcher is not None:
self._fetched_pkg = fetcher.pkg_path
if self._default_exit(fetcher) != os.EX_OK:
self._async_unlock_builddir(returncode=self.returncode)
@@ -209,7 +212,8 @@ class Binpkg(CompositeTask):
# This gives bashrc users an opportunity to do various things
# such as remove binary packages after they're installed.
- self.settings["PORTAGE_BINPKG_FILE"] = pkg_path
+ if pkg_path is not None:
+ self.settings["PORTAGE_BINPKG_FILE"] = pkg_path
self._pkg_path = pkg_path
logfile = self.settings.get("PORTAGE_LOG_FILE")
@@ -245,6 +249,13 @@ class Binpkg(CompositeTask):
self._async_unlock_builddir(returncode=self.returncode)
return
+ self._start_task(
+ AsyncTaskFuture(future=self._unpack_metadata()),
+ self._unpack_metadata_exit)
+
+ @coroutine
+ def _unpack_metadata(self):
+
dir_path = self.settings['PORTAGE_BUILDDIR']
infloc = self._infloc
@@ -260,8 +271,7 @@ class Binpkg(CompositeTask):
portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1)
self._writemsg_level(">>> Extracting info\n")
- pkg_xpak = portage.xpak.tbz2(self._pkg_path)
- pkg_xpak.unpackinfo(infloc)
+ yield self._bintree.dbapi.unpack_metadata(self.settings, infloc)
check_missing_metadata = ("CATEGORY", "PF")
for k, v in zip(check_missing_metadata,
self._bintree.dbapi.aux_get(self.pkg.cpv, check_missing_metadata)):
@@ -295,11 +305,14 @@ class Binpkg(CompositeTask):
env_extractor = BinpkgEnvExtractor(background=self.background,
scheduler=self.scheduler, settings=self.settings)
-
- self._start_task(env_extractor, self._env_extractor_exit)
-
- def _env_extractor_exit(self, env_extractor):
- if self._default_exit(env_extractor) != os.EX_OK:
+ env_extractor.start()
+ yield env_extractor.async_wait()
+ if env_extractor.returncode != os.EX_OK:
+ raise portage.exception.PortageException('failed to extract environment for {}'.format(self.pkg.cpv))
+
+ def _unpack_metadata_exit(self, unpack_metadata):
+ if self._default_exit(unpack_metadata) != os.EX_OK:
+ unpack_metadata.future.result()
self._async_unlock_builddir(returncode=self.returncode)
return
@@ -316,18 +329,16 @@ class Binpkg(CompositeTask):
self._async_unlock_builddir(returncode=self.returncode)
return
- extractor = BinpkgExtractorAsync(background=self.background,
- env=self.settings.environ(),
- features=self.settings.features,
- image_dir=self._image_dir,
- pkg=self.pkg, pkg_path=self._pkg_path,
- logfile=self.settings.get("PORTAGE_LOG_FILE"),
- scheduler=self.scheduler)
self._writemsg_level(">>> Extracting %s\n" % self.pkg.cpv)
- self._start_task(extractor, self._extractor_exit)
-
- def _extractor_exit(self, extractor):
- if self._default_exit(extractor) != os.EX_OK:
+ self._start_task(
+ AsyncTaskFuture(future=self._bintree.dbapi.unpack_contents(
+ self.settings,
+ self._image_dir)),
+ self._unpack_contents_exit)
+
+ def _unpack_contents_exit(self, unpack_contents):
+ if self._default_exit(unpack_contents) != os.EX_OK:
+ unpack_contents.future.result()
self._writemsg_level("!!! Error Extracting '%s'\n" % \
self._pkg_path, noiselevel=-1, level=logging.ERROR)
self._async_unlock_builddir(returncode=self.returncode)
diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py
index af43a2e24..7fa3992e7 100644
--- a/lib/_emerge/Scheduler.py
+++ b/lib/_emerge/Scheduler.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2014 Gentoo Foundation
+# Copyright 1999-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from __future__ import division, print_function, unicode_literals
@@ -868,10 +868,11 @@ class Scheduler(PollScheduler):
if fetched:
bintree.inject(x.cpv, filename=fetched)
- tbz2_file = bintree.getname(x.cpv)
+
infloc = os.path.join(build_dir_path, "build-info")
ensure_dirs(infloc)
- portage.xpak.tbz2(tbz2_file).unpackinfo(infloc)
+ self._sched_iface.run_until_complete(
+ bintree.dbapi.unpack_metadata(settings, infloc))
ebuild_path = os.path.join(infloc, x.pf + ".ebuild")
settings.configdict["pkg"]["EMERGE_FROM"] = "binary"
settings.configdict["pkg"]["MERGE_TYPE"] = "binary"
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 705a3ff1c..6f815bff2 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2018 Gentoo Foundation
+# Copyright 1999-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from __future__ import division, print_function, unicode_literals
@@ -122,6 +122,23 @@ def action_build(emerge_config, trees=DeprecationWarning,
# before we get here, so warn if they're not (bug #267103).
chk_updated_cfg_files(settings['EROOT'], ['/etc/portage'])
+ quickpkg_direct = ("--usepkg" in emerge_config.opts and
+ emerge_config.opts.get('--quickpkg-direct', 'n') == 'y' and
+ emerge_config.target_config is not emerge_config.running_config)
+ if '--getbinpkg' in emerge_config.opts or quickpkg_direct:
+ kwargs = {}
+ if quickpkg_direct:
+ kwargs['add_repos'] = (emerge_config.running_config.trees['vartree'].dbapi,)
+
+ try:
+ emerge_config.target_config.trees['bintree'].populate(
+ getbinpkgs='--getbinpkg' in emerge_config.opts,
+ **kwargs)
+ except ParseError as e:
+ writemsg("\n\n!!!%s.\nSee make.conf(5) for more info.\n"
+ % e, noiselevel=-1)
+ return 1
+
# validate the state of the resume data
# so that we can make assumptions later.
for k in ("resume", "resume_backup"):
@@ -352,12 +369,17 @@ def action_build(emerge_config, trees=DeprecationWarning,
# instances need to load remote metadata if --getbinpkg
# is enabled. Use getbinpkg_refresh=False to use cached
# metadata, since the cache is already fresh.
- if "--getbinpkg" in emerge_config.opts:
+ if "--getbinpkg" in emerge_config.opts or quickpkg_direct:
for root_trees in emerge_config.trees.values():
+ kwargs = {}
+ if quickpkg_direct:
+ kwargs['add_repos'] = (emerge_config.running_config.trees['vartree'].dbapi,)
+
try:
root_trees["bintree"].populate(
getbinpkgs=True,
- getbinpkg_refresh=False)
+ getbinpkg_refresh=False,
+ **kwargs)
except ParseError as e:
writemsg("\n\n!!!%s.\nSee make.conf(5) for more info.\n"
% e, noiselevel=-1)
@@ -2898,9 +2920,16 @@ def run_action(emerge_config):
if (emerge_config.action in ('search', None) and
'--usepkg' in emerge_config.opts):
for mytrees in emerge_config.trees.values():
+ kwargs = {}
+ if (mytrees is emerge_config.target_config.trees and
+ emerge_config.target_config is not emerge_config.running_config and
+ emerge_config.opts.get('--quickpkg-direct', 'n') == 'y'):
+ kwargs['add_repos'] = (emerge_config.running_config.trees['vartree'].dbapi,)
+
try:
mytrees['bintree'].populate(
- getbinpkgs='--getbinpkg' in emerge_config.opts)
+ getbinpkgs='--getbinpkg' in emerge_config.opts,
+ **kwargs)
except ParseError as e:
writemsg('\n\n!!!%s.\nSee make.conf(5) for more info.\n'
% (e,), noiselevel=-1)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 1127a6234..6d8e73172 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -495,6 +495,7 @@ class _dynamic_depgraph_config(object):
self._backtrack_infos = {}
self._buildpkgonly_deps_unsatisfied = False
+ self._quickpkg_direct_deps_unsatisfied = False
self._autounmask = self.myparams['autounmask']
self._displayed_autounmask = False
self._success_without_autounmask = False
@@ -4526,6 +4527,16 @@ class depgraph(object):
self._dynamic_config._skip_restart = True
return False, myfavorites
+ if (self._frozen_config.myopts.get('--quickpkg-direct', 'n') == 'y' and
+ self._frozen_config.target_root is not self._frozen_config._running_root):
+ running_root = self._frozen_config._running_root.root
+ for node in self._dynamic_config.digraph:
+ if (isinstance(node, Package) and node.operation in ('merge', 'uninstall') and
+ node.root == running_root):
+ self._dynamic_config._quickpkg_direct_deps_unsatisfied = True
+ self._dynamic_config._skip_restart = True
+ return False, myfavorites
+
if (not self._dynamic_config._prune_rebuilds and
self._ignored_binaries_autounmask_backtrack()):
config = self._dynamic_config._backtrack_infos.setdefault("config", {})
@@ -9062,6 +9073,14 @@ class depgraph(object):
writemsg("!!! Cannot merge requested packages. "
"Merge deps and try again.\n\n", noiselevel=-1)
+ if self._dynamic_config._quickpkg_direct_deps_unsatisfied:
+ self._show_merge_list()
+ writemsg("\n!!! --quickpkg-direct requires all "
+ "dependencies to be merged for root '{}'.\n".format(
+ self._frozen_config._running_root.root), noiselevel=-1)
+ writemsg("!!! Cannot merge requested packages. "
+ "Merge deps and try again.\n\n", noiselevel=-1)
+
def saveNomergeFavorites(self):
"""Find atoms in favorites that are not in the mergelist and add them
to the world file if necessary."""
diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py
index 0d2c45a4f..8c72cdf9c 100644
--- a/lib/_emerge/main.py
+++ b/lib/_emerge/main.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2018 Gentoo Foundation
+# Copyright 1999-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from __future__ import print_function
@@ -637,6 +637,11 @@ def parse_opts(tmpcmdline, silent=False):
"action" : "store",
},
+ "--quickpkg-direct": {
+ "help": "Enable use of installed packages directly as binary packages",
+ "choices": y_or_n
+ },
+
"--quiet": {
"shortopt" : "-q",
"help" : "reduced or condensed output",