aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--lib/_emerge/Binpkg.py59
-rw-r--r--lib/_emerge/Scheduler.py7
-rw-r--r--lib/_emerge/actions.py37
-rw-r--r--lib/_emerge/depgraph.py19
-rw-r--r--lib/_emerge/main.py7
-rw-r--r--lib/portage/dbapi/__init__.py5
-rw-r--r--lib/portage/dbapi/bintree.py112
-rw-r--r--lib/portage/dbapi/vartree.py117
-rw-r--r--lib/portage/tests/emerge/test_simple.py3
-rw-r--r--man/emerge.119
10 files changed, 338 insertions, 47 deletions
diff --git a/lib/_emerge/Binpkg.py b/lib/_emerge/Binpkg.py
index f9cffa26d..b5a69f8e7 100644
--- a/lib/_emerge/Binpkg.py
+++ b/lib/_emerge/Binpkg.py
@@ -7,7 +7,6 @@ import _emerge.emergelog
from _emerge.EbuildPhase import EbuildPhase
from _emerge.BinpkgFetcher import BinpkgFetcher
from _emerge.BinpkgEnvExtractor import BinpkgEnvExtractor
-from _emerge.BinpkgExtractorAsync import BinpkgExtractorAsync
from _emerge.CompositeTask import CompositeTask
from _emerge.BinpkgVerifier import BinpkgVerifier
from _emerge.EbuildMerge import EbuildMerge
@@ -16,6 +15,7 @@ from _emerge.SpawnProcess import SpawnProcess
from portage.eapi import eapi_exports_replace_vars
from portage.util import ensure_dirs
from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
+from portage.util.futures.compat_coroutine import coroutine
import portage
from portage import os
from portage import shutil
@@ -135,11 +135,14 @@ class Binpkg(CompositeTask):
pkg = self.pkg
pkg_count = self.pkg_count
- fetcher = BinpkgFetcher(background=self.background,
- logfile=self.settings.get("PORTAGE_LOG_FILE"), pkg=self.pkg,
- pretend=self.opts.pretend, scheduler=self.scheduler)
+ fetcher = None
if self.opts.getbinpkg and self._bintree.isremote(pkg.cpv):
+
+ fetcher = BinpkgFetcher(background=self.background,
+ logfile=self.settings.get("PORTAGE_LOG_FILE"), pkg=self.pkg,
+ pretend=self.opts.pretend, scheduler=self.scheduler)
+
msg = " --- (%s of %s) Fetching Binary (%s::%s)" %\
(pkg_count.curval, pkg_count.maxval, pkg.cpv,
fetcher.pkg_path)
@@ -160,7 +163,7 @@ class Binpkg(CompositeTask):
# The fetcher only has a returncode when
# --getbinpkg is enabled.
- if fetcher.returncode is not None:
+ if fetcher is not None:
self._fetched_pkg = fetcher.pkg_path
if self._default_exit(fetcher) != os.EX_OK:
self._async_unlock_builddir(returncode=self.returncode)
@@ -209,7 +212,8 @@ class Binpkg(CompositeTask):
# This gives bashrc users an opportunity to do various things
# such as remove binary packages after they're installed.
- self.settings["PORTAGE_BINPKG_FILE"] = pkg_path
+ if pkg_path is not None:
+ self.settings["PORTAGE_BINPKG_FILE"] = pkg_path
self._pkg_path = pkg_path
logfile = self.settings.get("PORTAGE_LOG_FILE")
@@ -245,6 +249,13 @@ class Binpkg(CompositeTask):
self._async_unlock_builddir(returncode=self.returncode)
return
+ self._start_task(
+ AsyncTaskFuture(future=self._unpack_metadata()),
+ self._unpack_metadata_exit)
+
+ @coroutine
+ def _unpack_metadata(self):
+
dir_path = self.settings['PORTAGE_BUILDDIR']
infloc = self._infloc
@@ -260,8 +271,7 @@ class Binpkg(CompositeTask):
portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1)
self._writemsg_level(">>> Extracting info\n")
- pkg_xpak = portage.xpak.tbz2(self._pkg_path)
- pkg_xpak.unpackinfo(infloc)
+ yield self._bintree.dbapi.unpack_metadata(self.settings, infloc)
check_missing_metadata = ("CATEGORY", "PF")
for k, v in zip(check_missing_metadata,
self._bintree.dbapi.aux_get(self.pkg.cpv, check_missing_metadata)):
@@ -295,11 +305,14 @@ class Binpkg(CompositeTask):
env_extractor = BinpkgEnvExtractor(background=self.background,
scheduler=self.scheduler, settings=self.settings)
-
- self._start_task(env_extractor, self._env_extractor_exit)
-
- def _env_extractor_exit(self, env_extractor):
- if self._default_exit(env_extractor) != os.EX_OK:
+ env_extractor.start()
+ yield env_extractor.async_wait()
+ if env_extractor.returncode != os.EX_OK:
+ raise portage.exception.PortageException('failed to extract environment for {}'.format(self.pkg.cpv))
+
+ def _unpack_metadata_exit(self, unpack_metadata):
+ if self._default_exit(unpack_metadata) != os.EX_OK:
+ unpack_metadata.future.result()
self._async_unlock_builddir(returncode=self.returncode)
return
@@ -316,18 +329,16 @@ class Binpkg(CompositeTask):
self._async_unlock_builddir(returncode=self.returncode)
return
- extractor = BinpkgExtractorAsync(background=self.background,
- env=self.settings.environ(),
- features=self.settings.features,
- image_dir=self._image_dir,
- pkg=self.pkg, pkg_path=self._pkg_path,
- logfile=self.settings.get("PORTAGE_LOG_FILE"),
- scheduler=self.scheduler)
self._writemsg_level(">>> Extracting %s\n" % self.pkg.cpv)
- self._start_task(extractor, self._extractor_exit)
-
- def _extractor_exit(self, extractor):
- if self._default_exit(extractor) != os.EX_OK:
+ self._start_task(
+ AsyncTaskFuture(future=self._bintree.dbapi.unpack_contents(
+ self.settings,
+ self._image_dir)),
+ self._unpack_contents_exit)
+
+ def _unpack_contents_exit(self, unpack_contents):
+ if self._default_exit(unpack_contents) != os.EX_OK:
+ unpack_contents.future.result()
self._writemsg_level("!!! Error Extracting '%s'\n" % \
self._pkg_path, noiselevel=-1, level=logging.ERROR)
self._async_unlock_builddir(returncode=self.returncode)
diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py
index af43a2e24..7fa3992e7 100644
--- a/lib/_emerge/Scheduler.py
+++ b/lib/_emerge/Scheduler.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2014 Gentoo Foundation
+# Copyright 1999-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from __future__ import division, print_function, unicode_literals
@@ -868,10 +868,11 @@ class Scheduler(PollScheduler):
if fetched:
bintree.inject(x.cpv, filename=fetched)
- tbz2_file = bintree.getname(x.cpv)
+
infloc = os.path.join(build_dir_path, "build-info")
ensure_dirs(infloc)
- portage.xpak.tbz2(tbz2_file).unpackinfo(infloc)
+ self._sched_iface.run_until_complete(
+ bintree.dbapi.unpack_metadata(settings, infloc))
ebuild_path = os.path.join(infloc, x.pf + ".ebuild")
settings.configdict["pkg"]["EMERGE_FROM"] = "binary"
settings.configdict["pkg"]["MERGE_TYPE"] = "binary"
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 705a3ff1c..6f815bff2 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2018 Gentoo Foundation
+# Copyright 1999-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from __future__ import division, print_function, unicode_literals
@@ -122,6 +122,23 @@ def action_build(emerge_config, trees=DeprecationWarning,
# before we get here, so warn if they're not (bug #267103).
chk_updated_cfg_files(settings['EROOT'], ['/etc/portage'])
+ quickpkg_direct = ("--usepkg" in emerge_config.opts and
+ emerge_config.opts.get('--quickpkg-direct', 'n') == 'y' and
+ emerge_config.target_config is not emerge_config.running_config)
+ if '--getbinpkg' in emerge_config.opts or quickpkg_direct:
+ kwargs = {}
+ if quickpkg_direct:
+ kwargs['add_repos'] = (emerge_config.running_config.trees['vartree'].dbapi,)
+
+ try:
+ emerge_config.target_config.trees['bintree'].populate(
+ getbinpkgs='--getbinpkg' in emerge_config.opts,
+ **kwargs)
+ except ParseError as e:
+ writemsg("\n\n!!!%s.\nSee make.conf(5) for more info.\n"
+ % e, noiselevel=-1)
+ return 1
+
# validate the state of the resume data
# so that we can make assumptions later.
for k in ("resume", "resume_backup"):
@@ -352,12 +369,17 @@ def action_build(emerge_config, trees=DeprecationWarning,
# instances need to load remote metadata if --getbinpkg
# is enabled. Use getbinpkg_refresh=False to use cached
# metadata, since the cache is already fresh.
- if "--getbinpkg" in emerge_config.opts:
+ if "--getbinpkg" in emerge_config.opts or quickpkg_direct:
for root_trees in emerge_config.trees.values():
+ kwargs = {}
+ if quickpkg_direct:
+ kwargs['add_repos'] = (emerge_config.running_config.trees['vartree'].dbapi,)
+
try:
root_trees["bintree"].populate(
getbinpkgs=True,
- getbinpkg_refresh=False)
+ getbinpkg_refresh=False,
+ **kwargs)
except ParseError as e:
writemsg("\n\n!!!%s.\nSee make.conf(5) for more info.\n"
% e, noiselevel=-1)
@@ -2898,9 +2920,16 @@ def run_action(emerge_config):
if (emerge_config.action in ('search', None) and
'--usepkg' in emerge_config.opts):
for mytrees in emerge_config.trees.values():
+ kwargs = {}
+ if (mytrees is emerge_config.target_config.trees and
+ emerge_config.target_config is not emerge_config.running_config and
+ emerge_config.opts.get('--quickpkg-direct', 'n') == 'y'):
+ kwargs['add_repos'] = (emerge_config.running_config.trees['vartree'].dbapi,)
+
try:
mytrees['bintree'].populate(
- getbinpkgs='--getbinpkg' in emerge_config.opts)
+ getbinpkgs='--getbinpkg' in emerge_config.opts,
+ **kwargs)
except ParseError as e:
writemsg('\n\n!!!%s.\nSee make.conf(5) for more info.\n'
% (e,), noiselevel=-1)
diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py
index 1127a6234..6d8e73172 100644
--- a/lib/_emerge/depgraph.py
+++ b/lib/_emerge/depgraph.py
@@ -495,6 +495,7 @@ class _dynamic_depgraph_config(object):
self._backtrack_infos = {}
self._buildpkgonly_deps_unsatisfied = False
+ self._quickpkg_direct_deps_unsatisfied = False
self._autounmask = self.myparams['autounmask']
self._displayed_autounmask = False
self._success_without_autounmask = False
@@ -4526,6 +4527,16 @@ class depgraph(object):
self._dynamic_config._skip_restart = True
return False, myfavorites
+ if (self._frozen_config.myopts.get('--quickpkg-direct', 'n') == 'y' and
+ self._frozen_config.target_root is not self._frozen_config._running_root):
+ running_root = self._frozen_config._running_root.root
+ for node in self._dynamic_config.digraph:
+ if (isinstance(node, Package) and node.operation in ('merge', 'uninstall') and
+ node.root == running_root):
+ self._dynamic_config._quickpkg_direct_deps_unsatisfied = True
+ self._dynamic_config._skip_restart = True
+ return False, myfavorites
+
if (not self._dynamic_config._prune_rebuilds and
self._ignored_binaries_autounmask_backtrack()):
config = self._dynamic_config._backtrack_infos.setdefault("config", {})
@@ -9062,6 +9073,14 @@ class depgraph(object):
writemsg("!!! Cannot merge requested packages. "
"Merge deps and try again.\n\n", noiselevel=-1)
+ if self._dynamic_config._quickpkg_direct_deps_unsatisfied:
+ self._show_merge_list()
+ writemsg("\n!!! --quickpkg-direct requires all "
+ "dependencies to be merged for root '{}'.\n".format(
+ self._frozen_config._running_root.root), noiselevel=-1)
+ writemsg("!!! Cannot merge requested packages. "
+ "Merge deps and try again.\n\n", noiselevel=-1)
+
def saveNomergeFavorites(self):
"""Find atoms in favorites that are not in the mergelist and add them
to the world file if necessary."""
diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py
index 0d2c45a4f..8c72cdf9c 100644
--- a/lib/_emerge/main.py
+++ b/lib/_emerge/main.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2018 Gentoo Foundation
+# Copyright 1999-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from __future__ import print_function
@@ -637,6 +637,11 @@ def parse_opts(tmpcmdline, silent=False):
"action" : "store",
},
+ "--quickpkg-direct": {
+ "help": "Enable use of installed packages directly as binary packages",
+ "choices": y_or_n
+ },
+
"--quiet": {
"shortopt" : "-q",
"help" : "reduced or condensed output",
diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py
index 80f8a689f..37728714e 100644
--- a/lib/portage/dbapi/__init__.py
+++ b/lib/portage/dbapi/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2018 Gentoo Foundation
+# Copyright 1998-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from __future__ import unicode_literals
@@ -32,8 +32,7 @@ class dbapi(object):
_use_mutable = False
_known_keys = frozenset(x for x in auxdbkeys
if not x.startswith("UNUSED_0"))
- _pkg_str_aux_keys = ("BUILD_TIME", "EAPI", "BUILD_ID",
- "KEYWORDS", "SLOT", "repository")
+ _pkg_str_aux_keys = ("EAPI", "KEYWORDS", "SLOT", "repository")
def __init__(self):
pass
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index ba21e6d23..9d3ea039b 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2018 Gentoo Foundation
+# Copyright 1998-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from __future__ import unicode_literals
@@ -7,6 +7,7 @@ __all__ = ["bindbapi", "binarytree"]
import portage
portage.proxy.lazyimport.lazyimport(globals(),
+ '_emerge.BinpkgExtractorAsync:BinpkgExtractorAsync',
'portage.checksum:get_valid_checksum_keys,perform_multiple_checksums,' + \
'verify_all,_apply_hash_filter,_hash_filter',
'portage.dbapi.dep_expand:dep_expand',
@@ -18,6 +19,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.util:atomic_ofstream,ensure_dirs,normalize_path,' + \
'writemsg,writemsg_stdout',
'portage.util.path:first_existing',
+ 'portage.util._async.SchedulerInterface:SchedulerInterface',
'portage.util._urlopen:urlopen@_urlopen,have_pep_476@_have_pep_476',
'portage.versions:best,catpkgsplit,catsplit,_pkg_str',
)
@@ -30,6 +32,9 @@ from portage.exception import AlarmSignal, InvalidData, InvalidPackageName, \
ParseError, PermissionDenied, PortageException
from portage.localization import _
from portage.package.ebuild.profile_iuse import iter_iuse_vars
+from portage.util.futures import asyncio
+from portage.util.futures.compat_coroutine import coroutine
+from portage.util.futures.executor.fork import ForkExecutor
from portage import _movefile
from portage import os
from portage import _encodings
@@ -70,6 +75,8 @@ class UseCachedCopyOfRemoteIndex(Exception):
class bindbapi(fakedbapi):
_known_keys = frozenset(list(fakedbapi._known_keys) + \
["CHOST", "repository", "USE"])
+ _pkg_str_aux_keys = fakedbapi._pkg_str_aux_keys + ("BUILD_ID", "BUILD_TIME", "_mtime_")
+
def __init__(self, mybintree=None, **kwargs):
# Always enable multi_instance mode for bindbapi indexing. This
# does not affect the local PKGDIR file layout, since that is
@@ -142,7 +149,10 @@ class bindbapi(fakedbapi):
return [aux_cache.get(x, "") for x in wants]
mysplit = mycpv.split("/")
mylist = []
- if not self.bintree._remotepkgs or \
+ add_pkg = self.bintree._additional_pkgs.get(instance_key)
+ if add_pkg is not None:
+ return add_pkg._db.aux_get(add_pkg, wants)
+ elif not self.bintree._remotepkgs or \
not self.bintree.isremote(mycpv):
try:
tbz2_path = self.bintree._pkg_paths[instance_key]
@@ -218,6 +228,73 @@ class bindbapi(fakedbapi):
# inject will clear stale caches via cpv_inject.
self.bintree.inject(cpv, filename=tbz2path)
+
+ @coroutine
+ def unpack_metadata(self, pkg, dest_dir):
+ """
+ Unpack package metadata to a directory. This method is a coroutine.
+
+ @param pkg: package to unpack
+ @type pkg: _pkg_str or portage.config
+ @param dest_dir: destination directory
+ @type dest_dir: str
+ """
+ loop = asyncio._wrap_loop()
+ if isinstance(pkg, _pkg_str):
+ cpv = pkg
+ else:
+ cpv = pkg.mycpv
+ key = self._instance_key(cpv)
+ add_pkg = self.bintree._additional_pkgs.get(key)
+ if add_pkg is not None:
+ yield add_pkg._db.unpack_metadata(pkg, dest_dir)
+ else:
+ tbz2_file = self.bintree.getname(cpv)
+ yield loop.run_in_executor(ForkExecutor(loop=loop),
+ portage.xpak.tbz2(tbz2_file).unpackinfo, dest_dir)
+
+ @coroutine
+ def unpack_contents(self, pkg, dest_dir):
+ """
+ Unpack package contents to a directory. This method is a coroutine.
+
+ @param pkg: package to unpack
+ @type pkg: _pkg_str or portage.config
+ @param dest_dir: destination directory
+ @type dest_dir: str
+ """
+ loop = asyncio._wrap_loop()
+ if isinstance(pkg, _pkg_str):
+ settings = self.settings
+ cpv = pkg
+ else:
+ settings = pkg
+ cpv = settings.mycpv
+
+ pkg_path = self.bintree.getname(cpv)
+ if pkg_path is not None:
+
+ extractor = BinpkgExtractorAsync(
+ background=settings.get('PORTAGE_BACKGROUND') == '1',
+ env=settings.environ(),
+ features=settings.features,
+ image_dir=dest_dir,
+ pkg=cpv, pkg_path=pkg_path,
+ logfile=settings.get('PORTAGE_LOG_FILE'),
+ scheduler=SchedulerInterface(loop))
+
+ extractor.start()
+ yield extractor.async_wait()
+ if extractor.returncode != os.EX_OK:
+ raise PortageException("Error Extracting '{}'".format(pkg_path))
+
+ else:
+ instance_key = self._instance_key(cpv)
+ add_pkg = self.bintree._additional_pkgs.get(instance_key)
+ if add_pkg is None:
+ raise portage.exception.PackageNotFound(cpv)
+ yield add_pkg._db.unpack_contents(pkg, dest_dir)
+
def cp_list(self, *pargs, **kwargs):
if not self.bintree.populated:
self.bintree.populate()
@@ -261,6 +338,7 @@ class bindbapi(fakedbapi):
return filesdict
+
class binarytree(object):
"this tree scans for a list of all packages available in PKGDIR"
def __init__(self, _unused=DeprecationWarning, pkgdir=None,
@@ -301,6 +379,7 @@ class binarytree(object):
self.tree = {}
self._remote_has_index = False
self._remotepkgs = None # remote metadata indexed by cpv
+ self._additional_pkgs = {}
self.invalids = []
self.settings = settings
self._pkg_paths = {}
@@ -511,7 +590,7 @@ class binarytree(object):
except PortageException:
pass
- def populate(self, getbinpkgs=False, getbinpkg_refresh=True):
+ def populate(self, getbinpkgs=False, getbinpkg_refresh=True, add_repos=()):
"""
Populates the binarytree with package metadata.
@@ -520,12 +599,14 @@ class binarytree(object):
@param getbinpkg_refresh: attempt to refresh the cache
of remote package metadata if getbinpkgs is also True
@type getbinpkg_refresh: bool
+ @param add_repos: additional binary package repositories
+ @type add_repos: sequence
"""
if self._populating:
return
- if not os.path.isdir(self.pkgdir) and not getbinpkgs:
+ if not os.path.isdir(self.pkgdir) and not (getbinpkgs or add_repos):
self.populated = True
return
@@ -557,6 +638,9 @@ class binarytree(object):
if pkgindex_lock:
unlockfile(pkgindex_lock)
+ if add_repos:
+ self._populate_additional(add_repos)
+
if getbinpkgs:
if not self.settings.get("PORTAGE_BINHOST"):
writemsg(_("!!! PORTAGE_BINHOST unset, but use is requested.\n"),
@@ -1066,6 +1150,16 @@ class binarytree(object):
self._merge_pkgindex_header(pkgindex.header,
self._pkgindex_header)
+ def _populate_additional(self, repos):
+ for repo in repos:
+ aux_keys = list(set(chain(repo._aux_cache_keys, repo._pkg_str_aux_keys)))
+ for cpv in repo.cpv_all():
+ metadata = dict(zip(aux_keys, repo.aux_get(cpv, aux_keys)))
+ pkg = _pkg_str(cpv, metadata=metadata, settings=repo.settings, db=repo)
+ instance_key = self.dbapi._instance_key(pkg)
+ self._additional_pkgs[instance_key] = pkg
+ self.dbapi.cpv_inject(pkg)
+
def inject(self, cpv, filename=None):
"""Add a freshly built package to the database. This updates
$PKGDIR/Packages with the new package metadata (including MD5).
@@ -1500,6 +1594,8 @@ class binarytree(object):
filename = self._pkg_paths.get(instance_key)
if filename is not None:
filename = os.path.join(self.pkgdir, filename)
+ elif instance_key in self._additional_pkgs:
+ return None
if filename is None:
if self._multi_instance:
@@ -1570,8 +1666,12 @@ class binarytree(object):
def isremote(self, pkgname):
"""Returns true if the package is kept remotely and it has not been
downloaded (or it is only partially downloaded)."""
- if (self._remotepkgs is None or
- self.dbapi._instance_key(pkgname) not in self._remotepkgs):
+ if self._remotepkgs is None:
+ return False
+ instance_key = self.dbapi._instance_key(pkgname)
+ if instance_key not in self._remotepkgs:
+ return False
+ elif instance_key in self._additional_pkgs:
return False
# Presence in self._remotepkgs implies that it's remote. When a
# package is downloaded, state is updated by self.inject().
diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py
index 603d58015..039e520d5 100644
--- a/lib/portage/dbapi/vartree.py
+++ b/lib/portage/dbapi/vartree.py
@@ -33,7 +33,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
'portage.util._compare_files:compare_files',
'portage.util.digraph:digraph',
'portage.util.env_update:env_update',
- 'portage.util.install_mask:install_mask_dir,InstallMask',
+ 'portage.util.install_mask:install_mask_dir,InstallMask,_raise_exc',
'portage.util.listdir:dircache,listdir',
'portage.util.movefile:movefile',
'portage.util.monotonic:monotonic',
@@ -71,6 +71,8 @@ from portage import _os_merge
from portage import _selinux_merge
from portage import _unicode_decode
from portage import _unicode_encode
+from portage.util.futures.compat_coroutine import coroutine
+from portage.util.futures.executor.fork import ForkExecutor
from ._VdbMetadataDelta import VdbMetadataDelta
from _emerge.EbuildBuildDir import EbuildBuildDir
@@ -80,8 +82,10 @@ from _emerge.MiscFunctionsProcess import MiscFunctionsProcess
from _emerge.SpawnProcess import SpawnProcess
from ._ContentsCaseSensitivityManager import ContentsCaseSensitivityManager
+import argparse
import errno
import fnmatch
+import functools
import gc
import grp
import io
@@ -128,6 +132,7 @@ class vardbapi(dbapi):
_aux_cache_keys_re = re.compile(r'^NEEDED\..*$')
_aux_multi_line_re = re.compile(r'^(CONTENTS|NEEDED\..*)$')
+ _pkg_str_aux_keys = dbapi._pkg_str_aux_keys + ("BUILD_ID", "BUILD_TIME", "_mtime_")
def __init__(self, _unused_param=DeprecationWarning,
categories=None, settings=None, vartree=None):
@@ -953,6 +958,110 @@ class vardbapi(dbapi):
pass
self._bump_mtime(cpv)
+ @coroutine
+ def unpack_metadata(self, pkg, dest_dir):
+ """
+ Unpack package metadata to a directory. This method is a coroutine.
+
+ @param pkg: package to unpack
+ @type pkg: _pkg_str or portage.config
+ @param dest_dir: destination directory
+ @type dest_dir: str
+ """
+ loop = asyncio._wrap_loop()
+ if not isinstance(pkg, portage.config):
+ cpv = pkg
+ else:
+ cpv = pkg.mycpv
+ dbdir = self.getpath(cpv)
+ def async_copy():
+ for parent, dirs, files in os.walk(dbdir, onerror=_raise_exc):
+ for key in files:
+ shutil.copy(os.path.join(parent, key),
+ os.path.join(dest_dir, key))
+ break
+ yield loop.run_in_executor(ForkExecutor(loop=loop), async_copy)
+
+ @coroutine
+ def unpack_contents(self, pkg, dest_dir,
+ include_config=None, include_unmodified_config=None):
+ """
+ Unpack package contents to a directory. This method is a coroutine.
+
+ This copies files from the installed system, in the same way
+ as the quickpkg(1) command. Default behavior for handling
+ of protected configuration files is controlled by the
+ QUICKPKG_DEFAULT_OPTS variable. The relevant quickpkg options
+ are --include-config and --include-unmodified-config. When
+ a configuration file is not included because it is protected,
+ an ewarn message is logged.
+
+ @param pkg: package to unpack
+ @type pkg: _pkg_str or portage.config
+ @param dest_dir: destination directory
+ @type dest_dir: str
+ @param include_config: Include all files protected by
+ CONFIG_PROTECT (as a security precaution, default is False
+ unless modified by QUICKPKG_DEFAULT_OPTS).
+ @type include_config: bool
+ @param include_unmodified_config: Include files protected by
+ CONFIG_PROTECT that have not been modified since installation
+ (as a security precaution, default is False unless modified
+ by QUICKPKG_DEFAULT_OPTS).
+ @type include_unmodified_config: bool
+ """
+ loop = asyncio._wrap_loop()
+ if not isinstance(pkg, portage.config):
+ settings = self.settings
+ cpv = pkg
+ else:
+ settings = pkg
+ cpv = settings.mycpv
+
+ scheduler = SchedulerInterface(loop)
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--include-config',
+ choices=('y', 'n'),
+ default='n')
+ parser.add_argument('--include-unmodified-config',
+ choices=('y', 'n'),
+ default='n')
+
+ # Method parameters may override QUICKPKG_DEFAULT_OPTS.
+ opts_list = portage.util.shlex_split(settings.get('QUICKPKG_DEFAULT_OPTS', ''))
+ if include_config is not None:
+ opts_list.append('--include-config={}'.format(
+ 'y' if include_config else 'n'))
+ if include_unmodified_config is not None:
+ opts_list.append('--include-unmodified-config={}'.format(
+ 'y' if include_unmodified_config else 'n'))
+
+ opts, args = parser.parse_known_args(opts_list)
+
+ tar_cmd = ('tar', '-x', '--xattrs', '--xattrs-include=*', '-C', dest_dir)
+ pr, pw = os.pipe()
+ proc = (yield asyncio.create_subprocess_exec(*tar_cmd, stdin=pr))
+ os.close(pr)
+ with os.fdopen(pw, 'wb', 0) as pw_file:
+ excluded_config_files = (yield loop.run_in_executor(ForkExecutor(loop=loop),
+ functools.partial(self._dblink(cpv).quickpkg,
+ pw_file,
+ include_config=opts.include_config == 'y',
+ include_unmodified_config=opts.include_unmodified_config == 'y')))
+ yield proc.wait()
+ if proc.returncode != os.EX_OK:
+ raise PortageException('command failed: {}'.format(tar_cmd))
+
+ if excluded_config_files:
+ log_lines = ([_("Config files excluded by QUICKPKG_DEFAULT_OPTS (see quickpkg(1) man page):")] +
+ ['\t{}'.format(name) for name in excluded_config_files])
+ out = io.StringIO()
+ for line in log_lines:
+ portage.elog.messages.ewarn(line, phase='install', key=cpv, out=out)
+ scheduler.output(out.getvalue(),
+ background=self.settings.get("PORTAGE_BACKGROUND") == "1",
+ log_path=settings.get("PORTAGE_LOG_FILE"))
+
def counter_tick(self, myroot=None, mycpv=None):
"""
@param myroot: ignored, self._eroot is used instead
@@ -1892,10 +2001,10 @@ class dblink(object):
@param include_config: Include all files protected by CONFIG_PROTECT
(as a security precaution, default is False).
@type include_config: bool
- @param include_config: Include files protected by CONFIG_PROTECT that
- have not been modified since installation (as a security precaution,
+ @param include_unmodified_config: Include files protected by CONFIG_PROTECT
+ that have not been modified since installation (as a security precaution,
default is False).
- @type include_config: bool
+ @type include_unmodified_config: bool
@rtype: list
@return: Paths of protected configuration files which have been omitted.
"""
diff --git a/lib/portage/tests/emerge/test_simple.py b/lib/portage/tests/emerge/test_simple.py
index 866521488..f5cd6f3d2 100644
--- a/lib/portage/tests/emerge/test_simple.py
+++ b/lib/portage/tests/emerge/test_simple.py
@@ -1,4 +1,4 @@
-# Copyright 2011-2018 Gentoo Foundation
+# Copyright 2011-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import subprocess
@@ -254,6 +254,7 @@ call_has_and_best_version() {
cross_eroot = os.path.join(cross_root, eprefix.lstrip(os.sep))
test_commands = (
+ emerge_cmd + ("--usepkgonly", "--root", cross_root, "--quickpkg-direct=y", "dev-libs/A"),
env_update_cmd,
portageq_cmd + ("envvar", "-v", "CONFIG_PROTECT", "EROOT",
"PORTAGE_CONFIGROOT", "PORTAGE_TMPDIR", "USERLAND"),
diff --git a/man/emerge.1 b/man/emerge.1
index 8d3e74074..8238515a6 100644
--- a/man/emerge.1
+++ b/man/emerge.1
@@ -1,4 +1,4 @@
-.TH "EMERGE" "1" "Jun 2019" "Portage VERSION" "Portage"
+.TH "EMERGE" "1" "Nov 2019" "Portage VERSION" "Portage"
.SH "NAME"
emerge \- Command\-line interface to the Portage system
.SH "SYNOPSIS"
@@ -829,6 +829,23 @@ B blocked by another package (unresolved conflict)
b blocked by another package (automatically resolved conflict)
.TE
.TP
+.BR "\-\-quickpkg\-direct < y | n >"
+Enable use of installed packages directly as binary packages. This is
+similar to using binary packages produced by \fBquickpkg\fR(1), but
+installed packages are used directly as though they are binary packages.
+This option only works in combination with the \fB\-\-root=DIR\fR option,
+and it comes with the caveat that packages are only allowed to be
+installed into the root that is specified by the \fB\-\-root=DIR\fR
+option (the other root which serves as a source of packages is
+assumed to be immutable during the entire operation).
+
+Default behavior for handling of protected configuration files is
+controlled by the \fBQUICKPKG_DEFAULT_OPTS\fR variable. The relevant
+quickpkg options are \fI\-\-include\-config\fR and
+\fI\-\-include\-unmodified\-config\fR (refer to the \fBquickpkg\fR(1)
+man page). When a configuration file is not included because it is
+protected, an ewarn message is logged.
+.TP
.BR "\-\-quiet [ y | n ]" ", " \-q
Results may vary, but the general outcome is a reduced or condensed
output from portage's displays.