diff options
Diffstat (limited to 'pym/portage')
75 files changed, 1327 insertions, 156 deletions
diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index d34e06ce7..c24abb0a5 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -494,7 +494,7 @@ _doebuild_manifest_exempt_depend = 0 _testing_eapis = frozenset(["4-python", "4-slot-abi", "5-progress", "5-hdepend"]) _deprecated_eapis = frozenset(["4_pre1", "3_pre2", "3_pre1", "5_pre1", "5_pre2"]) -_supported_eapis = frozenset([str(x) for x in range(portage.const.EAPI)] + list(_testing_eapis) + list(_deprecated_eapis)) +_supported_eapis = frozenset([str(x) for x in range(portage.const.EAPI + 1)] + list(_testing_eapis) + list(_deprecated_eapis)) def _eapi_is_deprecated(eapi): return eapi in _deprecated_eapis @@ -506,19 +506,7 @@ def eapi_is_supported(eapi): eapi = str(eapi) eapi = eapi.strip() - if _eapi_is_deprecated(eapi): - return True - - if eapi in _testing_eapis: - return True - - try: - eapi = int(eapi) - except ValueError: - eapi = -1 - if eapi < 0: - return False - return eapi <= portage.const.EAPI + return eapi in _supported_eapis # This pattern is specified by PMS section 7.3.1. _pms_eapi_re = re.compile(r"^[ \t]*EAPI=(['\"]?)([A-Za-z0-9+_.-]*)\1[ \t]*([ \t]#.*)?$") diff --git a/pym/portage/_emirrordist/FetchTask.py b/pym/portage/_emirrordist/FetchTask.py index 66c41c1a2..307c5bdda 100644 --- a/pym/portage/_emirrordist/FetchTask.py +++ b/pym/portage/_emirrordist/FetchTask.py @@ -1,6 +1,8 @@ -# Copyright 2013 Gentoo Foundation +# Copyright 2013-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import division + import collections import errno import logging @@ -242,7 +244,7 @@ class FetchTask(CompositeTask): remaining_tries = self.config.options.tries - len(self._tried_uris) if remaining_tries > 0: - if remaining_tries <= self.config.options.tries / 2: + if remaining_tries <= self.config.options.tries // 2: while self._primaryuri_stack: uri = self._primaryuri_stack.pop() if uri not in self._tried_uris: diff --git a/pym/portage/_global_updates.py b/pym/portage/_global_updates.py index dde726836..17dc0803b 100644 --- a/pym/portage/_global_updates.py +++ b/pym/portage/_global_updates.py @@ -118,7 +118,9 @@ def _do_global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): else: bindb = None - master_repo = portdb.getRepositoryName(portdb.porttree_root) + master_repo = portdb.repositories.mainRepo() + if master_repo is not None: + master_repo = master_repo.name if master_repo in repo_map: repo_map['DEFAULT'] = repo_map[master_repo] diff --git a/pym/portage/_sets/dbapi.py b/pym/portage/_sets/dbapi.py index 384fb3aa8..299cb8157 100644 --- a/pym/portage/_sets/dbapi.py +++ b/pym/portage/_sets/dbapi.py @@ -1,17 +1,21 @@ -# Copyright 2007-2012 Gentoo Foundation +# Copyright 2007-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import division + +import re import time from portage import os from portage.versions import best, catsplit, vercmp -from portage.dep import Atom +from portage.dep import Atom, use_reduce +from portage.exception import InvalidAtom from portage.localization import _ from portage._sets.base import PackageSet from portage._sets import SetConfigError, get_boolean import portage -__all__ = ["CategorySet", "DowngradeSet", +__all__ = ["CategorySet", "ChangedDepsSet", "DowngradeSet", "EverythingSet", "OwnerSet", "VariableSet"] class EverythingSet(PackageSet): @@ -456,3 +460,78 @@ class RebuiltBinaries(EverythingSet): bindb=trees["bintree"].dbapi) singleBuilder = classmethod(singleBuilder) + +class ChangedDepsSet(PackageSet): + + _operations = ["merge", "unmerge"] + + description = "Package set which contains all installed " + \ + "packages for which the vdb *DEPEND entries are outdated " + \ + "compared to corresponding portdb entries." + + def __init__(self, portdb=None, vardb=None): + super(ChangedDepsSet, self).__init__() + self._portdb = portdb + self._vardb = vardb + + def load(self): + depvars = ('RDEPEND', 'PDEPEND') + + # regexp used to match atoms using subslot operator := + subslot_repl_re = re.compile(r':[^[]*=') + + atoms = [] + for cpv in self._vardb.cpv_all(): + # no ebuild, no update :). + if not self._portdb.cpv_exists(cpv): + continue + + # USE flags used to build the ebuild and EAPI + # (needed for Atom & use_reduce()) + use, eapi = self._vardb.aux_get(cpv, ('USE', 'EAPI')) + usel = use.split() + + # function used to recursively process atoms in nested lists. + def clean_subslots(depatom, usel=None): + if isinstance(depatom, list): + # process the nested list. + return [clean_subslots(x, usel) for x in depatom] + else: + try: + # this can be either an atom or some special operator. + # in the latter case, we get InvalidAtom and pass it as-is. + a = Atom(depatom) + except InvalidAtom: + return depatom + else: + # if we're processing portdb, we need to evaluate USE flag + # dependency conditionals to make them match vdb. this + # requires passing the list of USE flags, so we reuse it + # as conditional for the operation as well. + if usel is not None: + a = a.evaluate_conditionals(usel) + + # replace slot operator := dependencies with plain := + # since we can't properly compare expanded slots + # in vardb to abstract slots in portdb. + return subslot_repl_re.sub(':=', a) + + # get all *DEPEND variables from vdb & portdb and compare them. + # we need to do some cleaning up & expansion to make matching + # meaningful since vdb dependencies are conditional-free. + vdbvars = [clean_subslots(use_reduce(x, uselist=usel, eapi=eapi)) + for x in self._vardb.aux_get(cpv, depvars)] + pdbvars = [clean_subslots(use_reduce(x, uselist=usel, eapi=eapi), usel) + for x in self._portdb.aux_get(cpv, depvars)] + + # if dependencies don't match, trigger the rebuild. + if vdbvars != pdbvars: + atoms.append('=%s' % cpv) + + self._setAtoms(atoms) + + def singleBuilder(cls, options, settings, trees): + return cls(portdb=trees["porttree"].dbapi, + vardb=trees["vartree"].dbapi) + + singleBuilder = classmethod(singleBuilder) diff --git a/pym/portage/cache/sqlite.py b/pym/portage/cache/sqlite.py index 42a239922..310ac9460 100644 --- a/pym/portage/cache/sqlite.py +++ b/pym/portage/cache/sqlite.py @@ -1,7 +1,7 @@ # Copyright 1999-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals +from __future__ import division, unicode_literals import re import sys @@ -174,7 +174,7 @@ class database(fs_template.FsBased): cursor.execute("PRAGMA page_size") page_size=int(cursor.fetchone()[0]) # number of pages, sqlite default is 2000 - cache_size = cache_bytes / page_size + cache_size = cache_bytes // page_size cursor.execute("PRAGMA cache_size = %d" % cache_size) cursor.execute("PRAGMA cache_size") actual_cache_size = int(cursor.fetchone()[0]) diff --git a/pym/portage/const.py b/pym/portage/const.py index 89d7ee29f..5f00fab87 100644 --- a/pym/portage/const.py +++ b/pym/portage/const.py @@ -1,5 +1,5 @@ # portage: Constants -# Copyright 1998-2013 Gentoo Foundation +# Copyright 1998-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import unicode_literals @@ -69,10 +69,10 @@ PORTAGE_BASE_PATH = PORTAGE_BASE # NOTE: Use realpath(__file__) so that python module symlinks in site-packages # are followed back to the real location of the whole portage installation. #PREFIX: below should work, but I'm not sure how it it affects other places -#PORTAGE_BASE_PATH = os.path.join(os.sep, os.sep.join(os.path.realpath( -# __file__.rstrip("co")).split(os.sep)[:-3])) +# NOTE: Please keep PORTAGE_BASE_PATH in one line to help substitutions. +#PORTAGE_BASE_PATH = os.path.join(os.sep, os.sep.join(os.path.realpath(__file__.rstrip("co")).split(os.sep)[:-3])) PORTAGE_BIN_PATH = PORTAGE_BASE_PATH + "/bin" -PORTAGE_PYM_PATH = PORTAGE_BASE_PATH + "/pym" +PORTAGE_PYM_PATH = os.path.realpath(os.path.join(__file__, '../..')) LOCALE_DATA_PATH = PORTAGE_BASE_PATH + "/locale" # FIXME: not used EBUILD_SH_BINARY = PORTAGE_BIN_PATH + "/ebuild.sh" MISC_SH_BINARY = PORTAGE_BIN_PATH + "/misc-functions.sh" @@ -94,6 +94,7 @@ PORTAGE_GROUPNAME = portagegroup PORTAGE_USERNAME = portageuser INVALID_ENV_FILE = "/etc/spork/is/not/valid/profile.env" +MERGING_IDENTIFIER = "-MERGING-" REPO_NAME_FILE = "repo_name" REPO_NAME_LOC = "profiles" + "/" + REPO_NAME_FILE @@ -291,6 +292,9 @@ SUPPORTED_BINPKG_FORMATS = ("tar", "rpm") # Time formats used in various places like metadata.chk. TIMESTAMP_FORMAT = "%a, %d %b %Y %H:%M:%S +0000" # to be used with time.gmtime() +# Top-level names of Python packages installed by Portage. +PORTAGE_PYM_PACKAGES = ("_emerge", "portage", "repoman") + # =========================================================================== # END OF CONSTANTS -- END OF CONSTANTS -- END OF CONSTANTS -- END OF CONSTANT # =========================================================================== diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py index a20a1e84f..34dfaa760 100644 --- a/pym/portage/dbapi/__init__.py +++ b/pym/portage/dbapi/__init__.py @@ -1,4 +1,4 @@ -# Copyright 1998-2013 Gentoo Foundation +# Copyright 1998-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import unicode_literals @@ -16,6 +16,8 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.versions:catsplit,catpkgsplit,vercmp,_pkg_str', ) +from portage.const import MERGING_IDENTIFIER + from portage import os from portage import auxdbkeys from portage.eapi import _get_eapi_attrs @@ -278,7 +280,7 @@ class dbapi(object): return True def invalidentry(self, mypath): - if '/-MERGING-' in mypath: + if "/" + MERGING_IDENTIFIER in mypath: if os.path.exists(mypath): writemsg(colorize("BAD", _("INCOMPLETE MERGE:"))+" %s\n" % mypath, noiselevel=-1) diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 040b546fa..deeb77950 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -1,7 +1,7 @@ # Copyright 1998-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals +from __future__ import division, unicode_literals __all__ = [ "vardbapi", "vartree", "dblink"] + \ @@ -25,6 +25,7 @@ portage.proxy.lazyimport.lazyimport(globals(), '_merge_unicode_error', '_spawn_phase', 'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs', 'portage.package.ebuild._ipc.QueryCommand:QueryCommand', + 'portage.process:find_binary', 'portage.util:apply_secpass_permissions,ConfigProtect,ensure_dirs,' + \ 'writemsg,writemsg_level,write_atomic,atomic_ofstream,writedict,' + \ 'grabdict,normalize_path,new_protect_filename', @@ -48,7 +49,7 @@ portage.proxy.lazyimport.lazyimport(globals(), ) from portage.const import CACHE_PATH, CONFIG_MEMORY_FILE, \ - PORTAGE_PACKAGE_ATOM, PRIVATE_PATH, VDB_PATH, EPREFIX, EPREFIX_LSTRIP, BASH_BINARY + MERGING_IDENTIFIER, PORTAGE_PACKAGE_ATOM, PRIVATE_PATH, VDB_PATH, EPREFIX, EPREFIX_LSTRIP, BASH_BINARY from portage.dbapi import dbapi from portage.exception import CommandNotFound, \ InvalidData, InvalidLocation, InvalidPackageName, \ @@ -108,7 +109,7 @@ class vardbapi(dbapi): _excluded_dirs = ["CVS", "lost+found"] _excluded_dirs = [re.escape(x) for x in _excluded_dirs] - _excluded_dirs = re.compile(r'^(\..*|-MERGING-.*|' + \ + _excluded_dirs = re.compile(r'^(\..*|' + MERGING_IDENTIFIER + '.*|' + \ "|".join(_excluded_dirs) + r')$') _aux_cache_version = "1" @@ -1058,7 +1059,7 @@ class vardbapi(dbapi): from md5 import new as _new_hash _hash_bits = 16 - _hex_chars = int(_hash_bits / 4) + _hex_chars = _hash_bits // 4 def __init__(self, vardb): self._vardb = vardb @@ -1519,7 +1520,7 @@ class dblink(object): self.dbroot = normalize_path(os.path.join(self._eroot, VDB_PATH)) self.dbcatdir = self.dbroot+"/"+cat self.dbpkgdir = self.dbcatdir+"/"+pkg - self.dbtmpdir = self.dbcatdir+"/-MERGING-"+pkg + self.dbtmpdir = self.dbcatdir+"/"+MERGING_IDENTIFIER+pkg self.dbdir = self.dbpkgdir self.settings = mysettings self._verbose = self.settings.get("PORTAGE_VERBOSE") == "1" @@ -3746,7 +3747,7 @@ class dblink(object): break relative_path = parent[srcroot_len:] - dirlist.append(os.path.join("/", relative_path)) + dirlist.append(os.path.join(destroot, relative_path)) for fname in files: try: @@ -3872,8 +3873,7 @@ class dblink(object): msg = textwrap.wrap(msg, 70) msg.append("") for f in rofilesystems: - msg.append("\t%s" % os.path.join(destroot, - f.lstrip(os.path.sep))) + msg.append("\t%s" % f) msg.append("") self._elog("eerror", "preinst", msg) @@ -5035,6 +5035,15 @@ class dblink(object): quickpkg_binary = os.path.join(self.settings["PORTAGE_BIN_PATH"], "quickpkg") + if not os.access(quickpkg_binary, os.X_OK): + # If not running from the source tree, use PATH. + quickpkg_binary = find_binary("quickpkg") + if quickpkg_binary is None: + self._display_merge( + _("%s: command not found") % "quickpkg", + level=logging.ERROR, noiselevel=-1) + return 127 + # Let quickpkg inherit the global vartree config's env. env = dict(self.vartree.settings.items()) env["__PORTAGE_INHERIT_VARDB_LOCK"] = "1" diff --git a/pym/portage/dep/_slot_operator.py b/pym/portage/dep/_slot_operator.py index 7b6444403..8b67fc569 100644 --- a/pym/portage/dep/_slot_operator.py +++ b/pym/portage/dep/_slot_operator.py @@ -1,4 +1,4 @@ -# Copyright 2012-2013 Gentoo Foundation +# Copyright 2012-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import unicode_literals @@ -67,6 +67,11 @@ def evaluate_slot_operator_equal_deps(settings, use, trees): return result def _eval_deps(dep_struct, vardbs): + # TODO: we'd use a better || () handling, i.e. || ( A:= B:= ) with both A + # and B installed should record subslot on A only since the package is + # supposed to link against that anyway, and we have no guarantee that B + # has matching ABI. + for i, x in enumerate(dep_struct): if isinstance(x, list): _eval_deps(x, vardbs) @@ -87,11 +92,15 @@ def _eval_deps(dep_struct, vardbs): dep_struct[i] = x break else: - # this dep could not be resolved, so remove the operator - # (user may be using package.provided and managing rebuilds - # manually) - if x.slot: - x = x.with_slot(x.slot) - else: - x = x.without_slot - dep_struct[i] = x + # this dep could not be resolved, possibilities include: + # 1. unsatisfied branch of || () dep, + # 2. package.provided, + # 3. --nodeps. + # + # just leave it as-is for now. this does not cause any special + # behavior while keeping the information in vdb -- necessary + # e.g. for @changed-deps to work properly. + # + # TODO: make it actually cause subslot rebuilds when switching + # || () branches. + pass diff --git a/pym/portage/dep/dep_check.py b/pym/portage/dep/dep_check.py index b79c5bc31..4386b5ed1 100644 --- a/pym/portage/dep/dep_check.py +++ b/pym/portage/dep/dep_check.py @@ -1,4 +1,4 @@ -# Copyright 2010-2013 Gentoo Foundation +# Copyright 2010-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import unicode_literals @@ -287,6 +287,7 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): unsat_use_non_installed = [] other_installed = [] other_installed_some = [] + other_installed_any_slot = [] other = [] # unsat_use_* must come after preferred_non_installed @@ -301,6 +302,7 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): unsat_use_non_installed, other_installed, other_installed_some, + other_installed_any_slot, other, ) @@ -414,16 +416,16 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): unsat_use_non_installed.append(this_choice) else: all_in_graph = True - for slot_atom in slot_map: + for atom in atoms: # New-style virtuals have zero cost to install. - if slot_atom.startswith("virtual/"): + if atom.blocker or atom.cp.startswith("virtual/"): continue # We check if the matched package has actually been # added to the digraph, in order to distinguish between # those packages and installed packages that may need # to be uninstalled in order to resolve blockers. - graph_matches = graph_db.match_pkgs(slot_atom) - if not graph_matches or graph_matches[-1] not in graph: + if not any(pkg in graph for pkg in + graph_db.match_pkgs(atom)): all_in_graph = False break circular_atom = None @@ -504,6 +506,14 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): other_installed.append(this_choice) elif some_installed: other_installed_some.append(this_choice) + + # Use Atom(atom.cp) for a somewhat "fuzzy" match, since + # the whole atom may be too specific. For example, see + # bug #522652, where using the whole atom leads to an + # unsatisfiable choice. + elif any(vardb.match(Atom(atom.cp)) for atom in atoms + if not atom.blocker): + other_installed_any_slot.append(this_choice) else: other.append(this_choice) diff --git a/pym/portage/dispatch_conf.py b/pym/portage/dispatch_conf.py index 479c19105..bf6c2ca94 100644 --- a/pym/portage/dispatch_conf.py +++ b/pym/portage/dispatch_conf.py @@ -208,4 +208,5 @@ def rcs_archive_post_process(archive): def file_archive_post_process(archive): """Rename the archive file with the .dist.new suffix to a .dist suffix""" - os.rename(archive + '.dist.new', archive + '.dist') + if os.path.exists(archive + '.dist.new'): + os.rename(archive + '.dist.new', archive + '.dist') diff --git a/pym/portage/emaint/main.py b/pym/portage/emaint/main.py index 6a17027b5..646883d73 100644 --- a/pym/portage/emaint/main.py +++ b/pym/portage/emaint/main.py @@ -98,10 +98,11 @@ def module_opts(module_controller, module): class TaskHandler(object): """Handles the running of the tasks it is given""" - def __init__(self, show_progress_bar=True, verbose=True, callback=None): + def __init__(self, show_progress_bar=True, verbose=True, callback=None, module_output=None): self.show_progress_bar = show_progress_bar self.verbose = verbose self.callback = callback + self.module_output = module_output self.isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty() self.progress_bar = ProgressBar(self.isatty, title="Emaint", max_desc_length=27) @@ -124,6 +125,7 @@ class TaskHandler(object): onProgress = None kwargs = { 'onProgress': onProgress, + 'module_output': self.module_output, # pass in a copy of the options so a module can not pollute or change # them for other tasks if there is more to do. 'options': options.copy() @@ -219,5 +221,5 @@ def emaint_main(myargv): # need to pass the parser options dict to the modules # so they are available if needed. task_opts = options.__dict__ - taskmaster = TaskHandler(callback=print_results) + taskmaster = TaskHandler(callback=print_results, module_output=sys.stdout) taskmaster.run_tasks(tasks, func, status, options=task_opts) diff --git a/pym/portage/emaint/module.py b/pym/portage/emaint/module.py index bf7d25fc5..07a0cb763 100644 --- a/pym/portage/emaint/module.py +++ b/pym/portage/emaint/module.py @@ -66,7 +66,7 @@ class Module(object): kid['is_imported'] = True except ImportError: raise - mod_class = getattr(module, kid['class']) + mod_class = getattr(module, kid['class']) return mod_class diff --git a/pym/portage/emaint/modules/binhost/__init__.py b/pym/portage/emaint/modules/binhost/__init__.py index c60e8bcb4..f2220e9d0 100644 --- a/pym/portage/emaint/modules/binhost/__init__.py +++ b/pym/portage/emaint/modules/binhost/__init__.py @@ -1,18 +1,18 @@ # Copyright 2005-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -"""Scan and generate metadata indexes for binary packages. -""" +doc = """Scan and generate metadata indexes for binary packages.""" +__doc__ = doc module_spec = { 'name': 'binhost', - 'description': __doc__, + 'description': doc, 'provides':{ 'module1': { 'name': "binhost", 'class': "BinhostHandler", - 'description': __doc__, + 'description': doc, 'functions': ['check', 'fix'], 'func_desc': {} } diff --git a/pym/portage/emaint/modules/config/__init__.py b/pym/portage/emaint/modules/config/__init__.py index f0585b39a..0277d3998 100644 --- a/pym/portage/emaint/modules/config/__init__.py +++ b/pym/portage/emaint/modules/config/__init__.py @@ -1,18 +1,18 @@ # Copyright 2005-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -"""Check and clean the config tracker list for uninstalled packages. -""" +doc = """Check and clean the config tracker list for uninstalled packages.""" +__doc__ = doc module_spec = { 'name': 'config', - 'description': __doc__, + 'description': doc, 'provides':{ 'module1': { 'name': "cleanconfmem", 'class': "CleanConfig", - 'description': __doc__, + 'description': doc, 'functions': ['check', 'fix'], 'func_desc': {} } diff --git a/pym/portage/emaint/modules/logs/__init__.py b/pym/portage/emaint/modules/logs/__init__.py index 0407efe2b..a7891fd5b 100644 --- a/pym/portage/emaint/modules/logs/__init__.py +++ b/pym/portage/emaint/modules/logs/__init__.py @@ -1,18 +1,18 @@ # Copyright 2005-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -"""Check and clean old logs in the PORT_LOGDIR. -""" +doc = """Check and clean old logs in the PORT_LOGDIR.""" +__doc__ = doc module_spec = { 'name': 'logs', - 'description': __doc__, + 'description': doc, 'provides':{ 'module1': { 'name': "logs", 'class': "CleanLogs", - 'description': __doc__, + 'description': doc, 'functions': ['check','clean'], 'func_desc': { 'clean': { diff --git a/pym/portage/emaint/modules/merges/__init__.py b/pym/portage/emaint/modules/merges/__init__.py new file mode 100644 index 000000000..bcb2ac8ee --- /dev/null +++ b/pym/portage/emaint/modules/merges/__init__.py @@ -0,0 +1,31 @@ +# Copyright 2005-2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +doc = """Scan for failed merges and fix them.""" +__doc__ = doc + + +module_spec = { + 'name': 'merges', + 'description': doc, + 'provides': { + 'merges': { + 'name': "merges", + 'class': "MergesHandler", + 'description': doc, + 'functions': ['check', 'fix', 'purge'], + 'func_desc': { + 'purge': { + 'short': '-P', 'long': '--purge-tracker', + 'help': 'Removes the list of previously failed merges.' + + ' WARNING: Only use this option if you plan on' + + ' manually fixing them or do not want them' + ' re-installed.', + 'status': "Removing %s", + 'action': 'store_true', + 'func': 'purge' + } + } + } + } +} diff --git a/pym/portage/emaint/modules/merges/merges.py b/pym/portage/emaint/modules/merges/merges.py new file mode 100644 index 000000000..1a67cb5df --- /dev/null +++ b/pym/portage/emaint/modules/merges/merges.py @@ -0,0 +1,290 @@ +# Copyright 2005-2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from _emerge.actions import load_emerge_config + +import portage +from portage import os, _unicode_encode +from portage.const import MERGING_IDENTIFIER, PORTAGE_BIN_PATH, PRIVATE_PATH, \ + VDB_PATH +from portage.dep import isvalidatom + +import shutil +import subprocess +import sys +import time + +class TrackingFile(object): + """File for keeping track of failed merges.""" + + + def __init__(self, tracking_path): + """ + Create a TrackingFile object. + + @param tracking_path: file path used to keep track of failed merges + @type tracking_path: String + """ + self._tracking_path = _unicode_encode(tracking_path) + + + def save(self, failed_pkgs): + """ + Save the specified packages that failed to merge. + + @param failed_pkgs: dictionary of failed packages + @type failed_pkgs: dict + """ + tracking_path = self._tracking_path + lines = ['%s %s' % (pkg, mtime) for pkg, mtime in failed_pkgs.items()] + portage.util.write_atomic(tracking_path, '\n'.join(lines)) + + + def load(self): + """ + Load previously failed merges. + + @rtype: dict + @return: dictionary of packages that failed to merge + """ + tracking_path = self._tracking_path + if not self.exists(): + return {} + failed_pkgs = {} + with open(tracking_path, 'r') as tracking_file: + for failed_merge in tracking_file: + pkg, mtime = failed_merge.strip().split() + failed_pkgs[pkg] = mtime + return failed_pkgs + + + def exists(self): + """ + Check if tracking file exists. + + @rtype: bool + @return: true if tracking file exists, false otherwise + """ + return os.path.exists(self._tracking_path) + + + def purge(self): + """Delete previously saved tracking file if one exists.""" + if self.exists(): + os.remove(self._tracking_path) + + + def __iter__(self): + """ + Provide an interator over failed merges. + + @return: iterator of packages that failed to merge + """ + return self.load().items().__iter__() + + +class MergesHandler(object): + """Handle failed package merges.""" + + short_desc = "Remove failed merges" + + @staticmethod + def name(): + return "merges" + + + def __init__(self): + """Create MergesHandler object.""" + eroot = portage.settings['EROOT'] + tracking_path = os.path.join(eroot, PRIVATE_PATH, 'failed-merges'); + self._tracking_file = TrackingFile(tracking_path) + self._vardb_path = os.path.join(eroot, VDB_PATH) + + + def can_progressbar(self, func): + return func == 'check' + + + def _scan(self, onProgress=None): + """ + Scan the file system for failed merges and return any found. + + @param onProgress: function to call for updating progress + @type onProgress: Function + @rtype: dict + @return: dictionary of packages that failed to merges + """ + failed_pkgs = {} + for cat in os.listdir(self._vardb_path): + pkgs_path = os.path.join(self._vardb_path, cat) + if not os.path.isdir(pkgs_path): + continue + pkgs = os.listdir(pkgs_path) + maxval = len(pkgs) + for i, pkg in enumerate(pkgs): + if onProgress: + onProgress(maxval, i+1) + if MERGING_IDENTIFIER in pkg: + mtime = int(os.stat(os.path.join(pkgs_path, pkg)).st_mtime) + pkg = os.path.join(cat, pkg) + failed_pkgs[pkg] = mtime + return failed_pkgs + + + def _failed_pkgs(self, onProgress=None): + """ + Return failed packages from both the file system and tracking file. + + @rtype: dict + @return: dictionary of packages that failed to merges + """ + failed_pkgs = self._scan(onProgress) + for pkg, mtime in self._tracking_file: + if pkg not in failed_pkgs: + failed_pkgs[pkg] = mtime + return failed_pkgs + + + def _remove_failed_dirs(self, failed_pkgs): + """ + Remove the directories of packages that failed to merge. + + @param failed_pkgs: failed packages whose directories to remove + @type failed_pkg: dict + """ + for failed_pkg in failed_pkgs: + pkg_path = os.path.join(self._vardb_path, failed_pkg) + # delete failed merge directory if it exists (it might not exist + # if loaded from tracking file) + if os.path.exists(pkg_path): + shutil.rmtree(pkg_path) + # TODO: try removing package CONTENTS to prevent orphaned + # files + + + def _get_pkg_atoms(self, failed_pkgs, pkg_atoms, pkg_invalid_entries): + """ + Get the package atoms for the specified failed packages. + + @param failed_pkgs: failed packages to iterate + @type failed_pkgs: dict + @param pkg_atoms: add package atoms to this set + @type pkg_atoms: set + @param pkg_invalid_entries: add any packages that are invalid to this set + @type pkg_invalid_entries: set + """ + + emerge_config = load_emerge_config() + portdb = emerge_config.target_config.trees['porttree'].dbapi + for failed_pkg in failed_pkgs: + # validate pkg name + pkg_name = '%s' % failed_pkg.replace(MERGING_IDENTIFIER, '') + pkg_atom = '=%s' % pkg_name + + if not isvalidatom(pkg_atom): + pkg_invalid_entries.add("'%s' is an invalid package atom." + % pkg_atom) + if not portdb.cpv_exists(pkg_name): + pkg_invalid_entries.add( + "'%s' does not exist in the portage tree." % pkg_name) + pkg_atoms.add(pkg_atom) + + + def _emerge_pkg_atoms(self, module_output, pkg_atoms): + """ + Emerge the specified packages atoms. + + @param module_output: output will be written to + @type module_output: Class + @param pkg_atoms: packages atoms to emerge + @type pkg_atoms: set + @rtype: list + @return: List of results + """ + # TODO: rewrite code to use portage's APIs instead of a subprocess + env = { + "FEATURES" : "-collision-protect -protect-owned", + "PATH" : os.environ["PATH"] + } + emerge_cmd = ( + portage._python_interpreter, + '-b', + os.path.join(PORTAGE_BIN_PATH, 'emerge'), + '--quiet', + '--oneshot', + '--complete-graph=y' + ) + results = [] + msg = 'Re-Emerging packages that failed to merge...\n' + if module_output: + module_output.write(msg) + else: + module_output = subprocess.PIPE + results.append(msg) + proc = subprocess.Popen(emerge_cmd + tuple(pkg_atoms), env=env, + stdout=module_output, stderr=sys.stderr) + output = proc.communicate()[0] + if output: + results.append(output) + if proc.returncode != os.EX_OK: + emerge_status = "Failed to emerge '%s'" % (' '.join(pkg_atoms)) + else: + emerge_status = "Successfully emerged '%s'" % (' '.join(pkg_atoms)) + results.append(emerge_status) + return results + + + def check(self, **kwargs): + """Check for failed merges.""" + onProgress = kwargs.get('onProgress', None) + failed_pkgs = self._failed_pkgs(onProgress) + errors = [] + for pkg, mtime in failed_pkgs.items(): + mtime_str = time.ctime(int(mtime)) + errors.append("'%s' failed to merge on '%s'" % (pkg, mtime_str)) + return errors + + + def fix(self, **kwargs): + """Attempt to fix any failed merges.""" + module_output = kwargs.get('module_output', None) + failed_pkgs = self._failed_pkgs() + if not failed_pkgs: + return ['No failed merges found.'] + + pkg_invalid_entries = set() + pkg_atoms = set() + self._get_pkg_atoms(failed_pkgs, pkg_atoms, pkg_invalid_entries) + if pkg_invalid_entries: + return pkg_invalid_entries + + try: + self._tracking_file.save(failed_pkgs) + except IOError as ex: + errors = ['Unable to save failed merges to tracking file: %s\n' + % str(ex)] + errors.append(', '.join(sorted(failed_pkgs))) + return errors + self._remove_failed_dirs(failed_pkgs) + results = self._emerge_pkg_atoms(module_output, pkg_atoms) + # list any new failed merges + for pkg in sorted(self._scan()): + results.append("'%s' still found as a failed merge." % pkg) + # reload config and remove successful packages from tracking file + emerge_config = load_emerge_config() + vardb = emerge_config.target_config.trees['vartree'].dbapi + still_failed_pkgs = {} + for pkg, mtime in failed_pkgs.items(): + pkg_name = '%s' % pkg.replace(MERGING_IDENTIFIER, '') + if not vardb.cpv_exists(pkg_name): + still_failed_pkgs[pkg] = mtime + self._tracking_file.save(still_failed_pkgs) + return results + + + def purge(self, **kwargs): + """Attempt to remove previously saved tracking file.""" + if not self._tracking_file.exists(): + return ['Tracking file not found.'] + self._tracking_file.purge() + return ['Removed tracking file.'] diff --git a/pym/portage/emaint/modules/move/__init__.py b/pym/portage/emaint/modules/move/__init__.py index d31d7b346..51624306c 100644 --- a/pym/portage/emaint/modules/move/__init__.py +++ b/pym/portage/emaint/modules/move/__init__.py @@ -1,18 +1,18 @@ # Copyright 2005-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -"""Perform package move updates for installed and binary packages. -""" +doc = """Perform package move updates for installed and binary packages.""" +__doc__ = doc module_spec = { 'name': 'move', - 'description': __doc__, + 'description': doc, 'provides':{ 'module1': { 'name': "moveinst", 'class': "MoveInstalled", - 'description': __doc__, + 'description': doc, 'options': ['check', 'fix'], 'functions': ['check', 'fix'], 'func_desc': { diff --git a/pym/portage/emaint/modules/move/move.py b/pym/portage/emaint/modules/move/move.py index ef674d47a..41ca167b7 100644 --- a/pym/portage/emaint/modules/move/move.py +++ b/pym/portage/emaint/modules/move/move.py @@ -13,8 +13,9 @@ class MoveHandler(object): self._tree = tree self._portdb = porttree.dbapi self._update_keys = Package._dep_keys + ("PROVIDE",) - self._master_repo = \ - self._portdb.getRepositoryName(self._portdb.porttree_root) + self._master_repo = self._portdb.repositories.mainRepo() + if self._master_repo is not None: + self._master_repo = self._master_repo.name def _grab_global_updates(self): from portage.update import grab_updates, parse_updates diff --git a/pym/portage/emaint/modules/resume/__init__.py b/pym/portage/emaint/modules/resume/__init__.py index 965e8f945..ebe4a37df 100644 --- a/pym/portage/emaint/modules/resume/__init__.py +++ b/pym/portage/emaint/modules/resume/__init__.py @@ -1,13 +1,13 @@ # Copyright 2005-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -"""Check and fix problems in the resume and/or resume_backup files. -""" +doc = """Check and fix problems in the resume and/or resume_backup files.""" +__doc__ = doc module_spec = { 'name': 'resume', - 'description': __doc__, + 'description': doc, 'provides':{ 'module1': { 'name': "cleanresume", diff --git a/pym/portage/emaint/modules/world/__init__.py b/pym/portage/emaint/modules/world/__init__.py index 3f62270ee..0af73d4e8 100644 --- a/pym/portage/emaint/modules/world/__init__.py +++ b/pym/portage/emaint/modules/world/__init__.py @@ -1,18 +1,18 @@ # Copyright 2005-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -"""Check and fix problems in the world file. -""" +doc = """Check and fix problems in the world file.""" +__doc__ = doc module_spec = { 'name': 'world', - 'description': __doc__, + 'description': doc, 'provides':{ 'module1':{ 'name': "world", 'class': "WorldHandler", - 'description': __doc__, + 'description': doc, 'functions': ['check', 'fix'], 'func_desc': {} } diff --git a/pym/portage/exception.py b/pym/portage/exception.py index 6fa5447a7..ef62e7ae1 100644 --- a/pym/portage/exception.py +++ b/pym/portage/exception.py @@ -84,6 +84,10 @@ class FileNotFound(InvalidLocation): class DirectoryNotFound(InvalidLocation): """A directory was not found when it was expected to exist""" +class IsADirectory(PortageException): + """A directory was found when it was expected to be a file""" + from errno import EISDIR as errno + class OperationNotPermitted(PortageException): """An operation was not permitted operating system""" from errno import EPERM as errno diff --git a/pym/portage/localization.py b/pym/portage/localization.py index e4d87b65b..2db4b7ada 100644 --- a/pym/portage/localization.py +++ b/pym/portage/localization.py @@ -2,10 +2,12 @@ # Copyright 2004-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import division + import locale import math -from portage import _unicode_decode +from portage import _encodings, _unicode_decode # We define this to make the transition easier for us. def _(mystr): @@ -36,4 +38,5 @@ def localized_size(num_bytes): # always round up, so that small files don't end up as '0 KiB' num_kib = math.ceil(num_bytes / 1024) - return locale.format('%d', num_kib, grouping=True) + ' KiB' + formatted_num = locale.format('%d', num_kib, grouping=True) + return (_unicode_decode(formatted_num, encoding=_encodings['stdio']) + ' KiB') diff --git a/pym/portage/mail.py b/pym/portage/mail.py index 723da04b8..11923eea6 100644 --- a/pym/portage/mail.py +++ b/pym/portage/mail.py @@ -12,7 +12,6 @@ import socket import sys -import time from portage import os from portage import _encodings @@ -49,6 +48,7 @@ def create_message(sender, recipient, subject, body, attachments=None): from email.header import Header from email.mime.base import MIMEBase as BaseMessage from email.mime.multipart import MIMEMultipart as MultipartMessage + from email.utils import formatdate if sys.hexversion < 0x3000000: sender = _unicode_encode(sender, @@ -91,8 +91,8 @@ def create_message(sender, recipient, subject, body, attachments=None): # input_bytes = s.encode(input_charset, errors) #UnicodeEncodeError: 'ascii' codec can't encode characters in position 0-9: ordinal not in range(128) mymessage["Subject"] = Header(_force_ascii_if_necessary(subject)) - mymessage["Date"] = time.strftime("%a, %d %b %Y %H:%M:%S %z") - + mymessage["Date"] = formatdate(localtime=True) + return mymessage def send_mail(mysettings, message): @@ -104,7 +104,7 @@ def send_mail(mysettings, message): mymailuser = "" mymailpasswd = "" myrecipient = "root@localhost" - + # Syntax for PORTAGE_ELOG_MAILURI (if defined): # address [[user:passwd@]mailserver[:port]] # where address: recipient address @@ -129,7 +129,7 @@ def send_mail(mysettings, message): mymailhost = myconndata else: myrecipient = mysettings.get("PORTAGE_ELOG_MAILURI", "") - + myfrom = message.get("From") if sys.hexversion < 0x3000000: @@ -174,4 +174,4 @@ def send_mail(mysettings, message): except socket.error as e: raise portage.exception.PortageException(_("!!! A network error occurred while trying to send logmail:\n%s\nSure you configured PORTAGE_ELOG_MAILURI correctly?") % str(e)) return - + diff --git a/pym/portage/news.py b/pym/portage/news.py index 408fb5c5f..0d72b000d 100644 --- a/pym/portage/news.py +++ b/pym/portage/news.py @@ -61,10 +61,12 @@ class NewsManager(object): self._dir_mode = 0o0074 self._mode_mask = 0o0000 - portdir = portdb.porttree_root - profiles_base = os.path.join(portdir, 'profiles') + os.path.sep + portdir = portdb.repositories.mainRepoLocation() + profiles_base = None + if portdir is not None: + profiles_base = os.path.join(portdir, 'profiles') + os.path.sep profile_path = None - if portdb.settings.profile_path: + if profiles_base is not None and portdb.settings.profile_path: profile_path = normalize_path( os.path.realpath(portdb.settings.profile_path)) if profile_path.startswith(profiles_base): diff --git a/pym/portage/output.py b/pym/portage/output.py index abe6b63cc..785168716 100644 --- a/pym/portage/output.py +++ b/pym/portage/output.py @@ -1,6 +1,8 @@ # Copyright 1998-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import division + __docformat__ = "epytext" import errno @@ -778,14 +780,14 @@ class TermProgressBar(ProgressBar): "<=>" + ((max_bar_width - bar_width) * " ") + "]") return image else: - percentage = int(100 * float(curval) / maxval) + percentage = 100 * curval // maxval max_bar_width = bar_space - 1 _percent = ("%d%% " % percentage).rjust(percentage_str_width) image = "%s%s" % (self._desc, _percent) if cols < min_columns: return image - offset = float(curval) / maxval + offset = curval / maxval bar_width = int(offset * max_bar_width) image = image + "[" + (bar_width * "=") + \ ">" + ((max_bar_width - bar_width) * " ") + "]" diff --git a/pym/portage/package/ebuild/config.py b/pym/portage/package/ebuild/config.py index fb4956d5d..6e578a9e1 100644 --- a/pym/portage/package/ebuild/config.py +++ b/pym/portage/package/ebuild/config.py @@ -37,7 +37,8 @@ from portage.dep import Atom, isvalidatom, match_from_list, use_reduce, _repo_se from portage.eapi import eapi_exports_AA, eapi_exports_merge_type, \ eapi_supports_prefix, eapi_exports_replace_vars, _get_eapi_attrs from portage.env.loaders import KeyValuePairFileLoader -from portage.exception import InvalidDependString, PortageException +from portage.exception import InvalidDependString, IsADirectory, \ + PortageException from portage.localization import _ from portage.output import colorize from portage.process import fakeroot_capable, sandbox_capable, macossandbox_capable @@ -556,8 +557,14 @@ class config(object): self.profile_path = locations_manager.profile_path self.user_profile_dir = locations_manager.user_profile_dir - packages_list = [grabfile_package(os.path.join(x, "packages"), - verify_eapi=True) for x in self.profiles] + try: + packages_list = [grabfile_package(os.path.join(x, "packages"), + verify_eapi=True) for x in self.profiles] + except IOError as e: + if e.errno == IsADirectory.errno: + raise IsADirectory(os.path.join(self.profile_path, + "packages")) + self.packages = tuple(stack_lists(packages_list, incremental=1)) # revmaskdict @@ -1359,6 +1366,7 @@ class config(object): previous_iuse = pkg_configdict.get("IUSE") previous_iuse_effective = pkg_configdict.get("IUSE_EFFECTIVE") previous_features = pkg_configdict.get("FEATURES") + previous_penv = self._penv aux_keys = self._setcpv_aux_keys @@ -1526,6 +1534,9 @@ class config(object): else: pkg_configdict['USE'] = self.puse + elif previous_penv: + has_changed = True + if has_changed: self.reset(keeping_pkg=1) diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py index 3c2167a03..8e55fe23e 100644 --- a/pym/portage/package/ebuild/doebuild.py +++ b/pym/portage/package/ebuild/doebuild.py @@ -46,8 +46,7 @@ from portage import auxdbkeys, bsd_chflags, \ unmerge, _encodings, _os_merge, \ _shell_quote, _unicode_decode, _unicode_encode from portage.const import EBUILD_SH_ENV_FILE, EBUILD_SH_ENV_DIR, \ - EBUILD_SH_BINARY, INVALID_ENV_FILE, MISC_SH_BINARY, \ - EPREFIX, MACOSSANDBOX_PROFILE + EBUILD_SH_BINARY, INVALID_ENV_FILE, MISC_SH_BINARY, PORTAGE_PYM_PACKAGES, EPREFIX, MACOSSANDBOX_PROFILE from portage.data import portage_gid, portage_uid, secpass, \ uid, userpriv_groups from portage.dbapi.porttree import _parse_uri_map @@ -2407,7 +2406,11 @@ def _prepare_self_update(settings): orig_pym_path = portage._pym_path portage._pym_path = os.path.join(base_path_tmp, "pym") - shutil.copytree(orig_pym_path, portage._pym_path, symlinks=True) + os.mkdir(portage._pym_path) + for pmod in PORTAGE_PYM_PACKAGES: + shutil.copytree(os.path.join(orig_pym_path, pmod), + os.path.join(portage._pym_path, pmod), + symlinks=True) for dir_path in (base_path_tmp, portage._bin_path, portage._pym_path): os.chmod(dir_path, 0o755) diff --git a/pym/portage/tests/__init__.py b/pym/portage/tests/__init__.py index 84e732a1c..708dd5934 100644 --- a/pym/portage/tests/__init__.py +++ b/pym/portage/tests/__init__.py @@ -25,8 +25,24 @@ import portage from portage import os from portage import _encodings from portage import _unicode_decode +from portage.const import (EPREFIX, GLOBAL_CONFIG_PATH, PORTAGE_BASE_PATH, + PORTAGE_BIN_PATH) from portage.util._argparse import ArgumentParser + +if portage._not_installed: + cnf_path = os.path.join(PORTAGE_BASE_PATH, 'cnf') + cnf_etc_path = cnf_path + cnf_bindir = PORTAGE_BIN_PATH + cnf_sbindir = cnf_bindir +else: + cnf_path = os.path.join(EPREFIX or '/', GLOBAL_CONFIG_PATH) + cnf_etc_path = os.path.join(EPREFIX or '/', 'etc') + cnf_eprefix = EPREFIX + cnf_bindir = os.path.join(EPREFIX or '/', 'usr', 'bin') + cnf_sbindir = os.path.join(EPREFIX or '/', 'usr', 'sbin') + + def main(): suite = unittest.TestSuite() basedir = os.path.dirname(os.path.realpath(__file__)) @@ -83,11 +99,11 @@ def getTestFromCommandLine(args, base_path): return result def getTestDirs(base_path): - TEST_FILE = b'__test__' + TEST_FILE = b'__test__.py' testDirs = [] # the os.walk help mentions relative paths as being quirky - # I was tired of adding dirs to the list, so now we add __test__ + # I was tired of adding dirs to the list, so now we add __test__.py # to each dir we want tested. for root, dirs, files in os.walk(base_path): try: @@ -178,6 +194,10 @@ class TestCase(unittest.TestCase): unittest.TestCase.__init__(self, *pargs, **kwargs) self.todo = False self.portage_skip = None + self.cnf_path = cnf_path + self.cnf_etc_path = cnf_etc_path + self.bindir = cnf_bindir + self.sbindir = cnf_sbindir def defaultTestResult(self): return TextTestResult() diff --git a/pym/portage/tests/bin/__test__ b/pym/portage/tests/bin/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/bin/__test__ +++ b/pym/portage/tests/bin/__test__.py diff --git a/pym/portage/tests/dbapi/__test__ b/pym/portage/tests/dbapi/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/dbapi/__test__ +++ b/pym/portage/tests/dbapi/__test__.py diff --git a/pym/portage/tests/dbapi/test_portdb_cache.py b/pym/portage/tests/dbapi/test_portdb_cache.py index 94af96eaf..f08d0f80b 100644 --- a/pym/portage/tests/dbapi/test_portdb_cache.py +++ b/pym/portage/tests/dbapi/test_portdb_cache.py @@ -8,8 +8,7 @@ import textwrap import portage from portage import os from portage import _unicode_decode -from portage.const import (BASH_BINARY, PORTAGE_BIN_PATH, - PORTAGE_PYM_PATH, USER_CONFIG_PATH) +from portage.const import (BASH_BINARY, PORTAGE_PYM_PATH, USER_CONFIG_PATH) from portage.tests import TestCase from portage.tests.resolver.ResolverPlayground import ResolverPlayground from portage.util import ensure_dirs @@ -38,7 +37,7 @@ class PortdbCacheTestCase(TestCase): portage_python = portage._python_interpreter egencache_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "egencache"), + os.path.join(self.bindir, "egencache"), "--repo", "test_repo", "--repositories-configuration", settings.repositories.config_string()) python_cmd = (portage_python, "-b", "-Wd", "-c") @@ -48,7 +47,7 @@ class PortdbCacheTestCase(TestCase): (lambda: not os.path.exists(md5_cache_dir),), python_cmd + (textwrap.dedent(""" import os, sys, portage - if portage.portdb.porttree_root in portage.portdb._pregen_auxdb: + if portage.portdb.repositories.mainRepoLocation() in portage.portdb._pregen_auxdb: sys.exit(1) """),), @@ -57,13 +56,13 @@ class PortdbCacheTestCase(TestCase): (lambda: os.path.exists(md5_cache_dir),), python_cmd + (textwrap.dedent(""" import os, sys, portage - if portage.portdb.porttree_root not in portage.portdb._pregen_auxdb: + if portage.portdb.repositories.mainRepoLocation() not in portage.portdb._pregen_auxdb: sys.exit(1) """),), python_cmd + (textwrap.dedent(""" import os, sys, portage from portage.cache.flat_hash import md5_database - if not isinstance(portage.portdb._pregen_auxdb[portage.portdb.porttree_root], md5_database): + if not isinstance(portage.portdb._pregen_auxdb[portage.portdb.repositories.mainRepoLocation()], md5_database): sys.exit(1) """),), @@ -74,13 +73,13 @@ class PortdbCacheTestCase(TestCase): (lambda: os.path.exists(md5_cache_dir),), python_cmd + (textwrap.dedent(""" import os, sys, portage - if portage.portdb.porttree_root not in portage.portdb._pregen_auxdb: + if portage.portdb.repositories.mainRepoLocation() not in portage.portdb._pregen_auxdb: sys.exit(1) """),), python_cmd + (textwrap.dedent(""" import os, sys, portage from portage.cache.flat_hash import md5_database - if not isinstance(portage.portdb._pregen_auxdb[portage.portdb.porttree_root], md5_database): + if not isinstance(portage.portdb._pregen_auxdb[portage.portdb.repositories.mainRepoLocation()], md5_database): sys.exit(1) """),), @@ -91,13 +90,13 @@ class PortdbCacheTestCase(TestCase): ("cache-formats = pms md5-dict", layout_conf_path,)))), (portage_python, "-b", "-Wd", "-Wi::DeprecationWarning", "-c") + (textwrap.dedent(""" import os, sys, portage - if portage.portdb.porttree_root not in portage.portdb._pregen_auxdb: + if portage.portdb.repositories.mainRepoLocation() not in portage.portdb._pregen_auxdb: sys.exit(1) """),), (portage_python, "-b", "-Wd", "-Wi::DeprecationWarning", "-c") + (textwrap.dedent(""" import os, sys, portage from portage.cache.metadata import database as pms_database - if not isinstance(portage.portdb._pregen_auxdb[portage.portdb.porttree_root], pms_database): + if not isinstance(portage.portdb._pregen_auxdb[portage.portdb.repositories.mainRepoLocation()], pms_database): sys.exit(1) """),), @@ -106,13 +105,13 @@ class PortdbCacheTestCase(TestCase): (BASH_BINARY, "-c", "rm %s" % portage._shell_quote(layout_conf_path)), python_cmd + (textwrap.dedent(""" import os, sys, portage - if portage.portdb.porttree_root not in portage.portdb._pregen_auxdb: + if portage.portdb.repositories.mainRepoLocation() not in portage.portdb._pregen_auxdb: sys.exit(1) """),), python_cmd + (textwrap.dedent(""" import os, sys, portage from portage.cache.flat_hash import md5_database - if not isinstance(portage.portdb._pregen_auxdb[portage.portdb.porttree_root], md5_database): + if not isinstance(portage.portdb._pregen_auxdb[portage.portdb.repositories.mainRepoLocation()], md5_database): sys.exit(1) """),), ) diff --git a/pym/portage/tests/dep/__test__ b/pym/portage/tests/dep/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/dep/__test__ +++ b/pym/portage/tests/dep/__test__.py diff --git a/pym/portage/tests/ebuild/__test__ b/pym/portage/tests/ebuild/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/ebuild/__test__ +++ b/pym/portage/tests/ebuild/__test__.py diff --git a/pym/portage/tests/ebuild/test_config.py b/pym/portage/tests/ebuild/test_config.py index 08e0a5dcf..20aac519a 100644 --- a/pym/portage/tests/ebuild/test_config.py +++ b/pym/portage/tests/ebuild/test_config.py @@ -1,13 +1,20 @@ # Copyright 2010-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + +import io +import tempfile + import portage -from portage import os +from portage import os, shutil, _encodings +from portage.const import USER_CONFIG_PATH from portage.dep import Atom from portage.package.ebuild.config import config from portage.package.ebuild._config.LicenseManager import LicenseManager from portage.tests import TestCase from portage.tests.resolver.ResolverPlayground import ResolverPlayground, ResolverPlaygroundTestCase +from portage.util import normalize_path class ConfigTestCase(TestCase): @@ -274,3 +281,65 @@ class ConfigTestCase(TestCase): self.assertEqual(test_case.test_success, True, test_case.fail_msg) finally: playground.cleanup() + + + def testSetCpv(self): + """ + Test the clone via constructor. + """ + + ebuilds = { + "dev-libs/A-1": {"IUSE": "static-libs"}, + "dev-libs/B-1": {"IUSE": "static-libs"}, + } + + env_files = { + "A" : ("USE=\"static-libs\"",) + } + + package_env = ( + "dev-libs/A A", + ) + + eprefix = normalize_path(tempfile.mkdtemp()) + playground = None + try: + user_config_dir = os.path.join(eprefix, USER_CONFIG_PATH) + os.makedirs(user_config_dir) + + with io.open(os.path.join(user_config_dir, "package.env"), + mode='w', encoding=_encodings['content']) as f: + for line in package_env: + f.write(line + "\n") + + env_dir = os.path.join(user_config_dir, "env") + os.makedirs(env_dir) + for k, v in env_files.items(): + with io.open(os.path.join(env_dir, k), mode='w', + encoding=_encodings['content']) as f: + for line in v: + f.write(line + "\n") + + playground = ResolverPlayground(eprefix=eprefix, ebuilds=ebuilds) + settings = config(clone=playground.settings) + + result = playground.run(["=dev-libs/A-1"]) + pkg, existing_node = result.depgraph._select_package( + playground.eroot, Atom("=dev-libs/A-1")) + settings.setcpv(pkg) + self.assertTrue("static-libs" in + settings["PORTAGE_USE"].split()) + + # Test bug #522362, where a USE=static-libs package.env + # setting leaked from one setcpv call to the next. + pkg, existing_node = result.depgraph._select_package( + playground.eroot, Atom("=dev-libs/B-1")) + settings.setcpv(pkg) + self.assertTrue("static-libs" not in + settings["PORTAGE_USE"].split()) + + finally: + if playground is None: + shutil.rmtree(eprefix) + else: + playground.cleanup() diff --git a/pym/portage/tests/emerge/__test__ b/pym/portage/tests/emerge/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/emerge/__test__ +++ b/pym/portage/tests/emerge/__test__.py diff --git a/pym/portage/tests/emerge/test_emerge_slot_abi.py b/pym/portage/tests/emerge/test_emerge_slot_abi.py index fd7ec0e6a..d1f2d9241 100644 --- a/pym/portage/tests/emerge/test_emerge_slot_abi.py +++ b/pym/portage/tests/emerge/test_emerge_slot_abi.py @@ -7,8 +7,7 @@ import sys import portage from portage import os from portage import _unicode_decode -from portage.const import (BASH_BINARY, PORTAGE_BIN_PATH, - PORTAGE_PYM_PATH, USER_CONFIG_PATH) +from portage.const import (BASH_BINARY, PORTAGE_PYM_PATH, USER_CONFIG_PATH) from portage.process import find_binary from portage.tests import TestCase from portage.tests.resolver.ResolverPlayground import ResolverPlayground @@ -70,9 +69,9 @@ class SlotAbiEmergeTestCase(TestCase): portage_python = portage._python_interpreter ebuild_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "ebuild")) + os.path.join(self.bindir, "ebuild")) emerge_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "emerge")) + os.path.join(self.bindir, "emerge")) test_ebuild = portdb.findname("dev-libs/dbus-glib-0.98") self.assertFalse(test_ebuild is None) diff --git a/pym/portage/tests/emerge/test_simple.py b/pym/portage/tests/emerge/test_simple.py index bf0af8bc8..0bb83ae4e 100644 --- a/pym/portage/tests/emerge/test_simple.py +++ b/pym/portage/tests/emerge/test_simple.py @@ -8,7 +8,7 @@ import portage from portage import os from portage import _unicode_decode from portage.const import (BASH_BINARY, PORTAGE_BASE_PATH, - PORTAGE_BIN_PATH, PORTAGE_PYM_PATH, USER_CONFIG_PATH) + PORTAGE_PYM_PATH, USER_CONFIG_PATH) from portage.process import find_binary from portage.tests import TestCase from portage.tests.resolver.ResolverPlayground import ResolverPlayground @@ -175,29 +175,29 @@ pkg_preinst() { portage_python = portage._python_interpreter dispatch_conf_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "dispatch-conf")) + os.path.join(self.sbindir, "dispatch-conf")) ebuild_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "ebuild")) + os.path.join(self.bindir, "ebuild")) egencache_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "egencache"), + os.path.join(self.bindir, "egencache"), "--repo", "test_repo", "--repositories-configuration", settings.repositories.config_string()) emerge_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "emerge")) + os.path.join(self.bindir, "emerge")) emaint_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "emaint")) + os.path.join(self.sbindir, "emaint")) env_update_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "env-update")) + os.path.join(self.sbindir, "env-update")) etc_update_cmd = (BASH_BINARY, - os.path.join(PORTAGE_BIN_PATH, "etc-update")) + os.path.join(self.sbindir, "etc-update")) fixpackages_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "fixpackages")) + os.path.join(self.sbindir, "fixpackages")) portageq_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "portageq")) + os.path.join(self.bindir, "portageq")) quickpkg_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "quickpkg")) + os.path.join(self.bindir, "quickpkg")) regenworld_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "regenworld")) + os.path.join(self.sbindir, "regenworld")) rm_binary = find_binary("rm") self.assertEqual(rm_binary is None, False, @@ -243,6 +243,12 @@ pkg_preinst() { emerge_cmd + ("--metadata",), rm_cmd + ("-rf", cachedir), emerge_cmd + ("--oneshot", "virtual/foo"), + lambda: self.assertFalse(os.path.exists( + os.path.join(pkgdir, "virtual", "foo-0.tbz2"))), + ({"FEATURES" : "unmerge-backup"},) + \ + emerge_cmd + ("--unmerge", "virtual/foo"), + lambda: self.assertTrue(os.path.exists( + os.path.join(pkgdir, "virtual", "foo-0.tbz2"))), emerge_cmd + ("--pretend", "dev-libs/A"), ebuild_cmd + (test_ebuild, "manifest", "clean", "package", "merge"), emerge_cmd + ("--pretend", "--tree", "--complete-graph", "dev-libs/A"), @@ -368,7 +374,7 @@ pkg_preinst() { for x in true_symlinks: os.symlink(true_binary, os.path.join(fake_bin, x)) for x in etc_symlinks: - os.symlink(os.path.join(PORTAGE_BASE_PATH, "cnf", x), + os.symlink(os.path.join(self.cnf_etc_path, x), os.path.join(eprefix, "etc", x)) with open(os.path.join(var_cache_edb, "counter"), 'wb') as f: f.write(b"100") @@ -395,6 +401,10 @@ move dev-util/git dev-vcs/git for args in test_commands: + if hasattr(args, '__call__'): + args() + continue + if isinstance(args[0], dict): local_env = env.copy() local_env.update(args[0]) diff --git a/pym/portage/tests/env/__test__ b/pym/portage/tests/env/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/env/__test__ +++ b/pym/portage/tests/env/__test__.py diff --git a/pym/portage/tests/env/config/__test__ b/pym/portage/tests/env/config/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/env/config/__test__ +++ b/pym/portage/tests/env/config/__test__.py diff --git a/pym/portage/tests/glsa/__test__ b/pym/portage/tests/glsa/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/glsa/__test__ +++ b/pym/portage/tests/glsa/__test__.py diff --git a/pym/portage/tests/glsa/test_security_set.py b/pym/portage/tests/glsa/test_security_set.py index edf567809..bf1f82b90 100644 --- a/pym/portage/tests/glsa/test_security_set.py +++ b/pym/portage/tests/glsa/test_security_set.py @@ -129,7 +129,8 @@ class SecuritySetTestCase(TestCase): try: portdb = playground.trees[playground.eroot]["porttree"].dbapi - glsa_dir = os.path.join(portdb.porttree_root, 'metadata', 'glsa') + glsa_dir = os.path.join( + portdb.repositories.mainRepoLocation(), 'metadata', 'glsa') portage.util.ensure_dirs(glsa_dir) for glsa in glsas: with io.open(os.path.join(glsa_dir, diff --git a/pym/portage/tests/lafilefixer/__test__ b/pym/portage/tests/lafilefixer/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/lafilefixer/__test__ +++ b/pym/portage/tests/lafilefixer/__test__.py diff --git a/pym/portage/tests/lazyimport/__test__ b/pym/portage/tests/lazyimport/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/lazyimport/__test__ +++ b/pym/portage/tests/lazyimport/__test__.py diff --git a/pym/portage/tests/lint/__test__ b/pym/portage/tests/lint/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/lint/__test__ +++ b/pym/portage/tests/lint/__test__.py diff --git a/pym/portage/tests/lint/test_compile_modules.py b/pym/portage/tests/lint/test_compile_modules.py index ce7e3fb90..4826cadb7 100644 --- a/pym/portage/tests/lint/test_compile_modules.py +++ b/pym/portage/tests/lint/test_compile_modules.py @@ -5,7 +5,7 @@ import errno import itertools import stat -from portage.const import PORTAGE_BIN_PATH, PORTAGE_PYM_PATH +from portage.const import PORTAGE_BIN_PATH, PORTAGE_PYM_PATH, PORTAGE_PYM_PACKAGES from portage.tests import TestCase from portage import os from portage import _encodings @@ -14,9 +14,11 @@ from portage import _unicode_decode, _unicode_encode class CompileModulesTestCase(TestCase): def testCompileModules(self): - for parent, _dirs, files in itertools.chain( - os.walk(PORTAGE_BIN_PATH), - os.walk(PORTAGE_PYM_PATH)): + iters = [os.walk(os.path.join(PORTAGE_PYM_PATH, x)) + for x in PORTAGE_PYM_PACKAGES] + iters.append(os.walk(PORTAGE_BIN_PATH)) + + for parent, _dirs, files in itertools.chain(*iters): parent = _unicode_decode(parent, encoding=_encodings['fs'], errors='strict') for x in files: diff --git a/pym/portage/tests/lint/test_import_modules.py b/pym/portage/tests/lint/test_import_modules.py index 34261f464..fcdcb3b33 100644 --- a/pym/portage/tests/lint/test_import_modules.py +++ b/pym/portage/tests/lint/test_import_modules.py @@ -1,7 +1,9 @@ # Copyright 2011-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from portage.const import PORTAGE_PYM_PATH +from itertools import chain + +from portage.const import PORTAGE_PYM_PATH, PORTAGE_PYM_PACKAGES from portage.tests import TestCase from portage import os from portage import _encodings @@ -13,7 +15,9 @@ class ImportModulesTestCase(TestCase): expected_failures = frozenset(( )) - for mod in self._iter_modules(PORTAGE_PYM_PATH): + iters = (self._iter_modules(os.path.join(PORTAGE_PYM_PATH, x)) + for x in PORTAGE_PYM_PACKAGES) + for mod in chain(*iters): try: __import__(mod) except ImportError as e: diff --git a/pym/portage/tests/locks/__test__ b/pym/portage/tests/locks/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/locks/__test__ +++ b/pym/portage/tests/locks/__test__.py diff --git a/pym/portage/tests/news/__test__ b/pym/portage/tests/news/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/news/__test__ +++ b/pym/portage/tests/news/__test__.py diff --git a/pym/portage/tests/process/__test__ b/pym/portage/tests/process/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/process/__test__ +++ b/pym/portage/tests/process/__test__.py diff --git a/pym/portage/tests/repoman/__test__ b/pym/portage/tests/repoman/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/repoman/__test__ +++ b/pym/portage/tests/repoman/__test__.py diff --git a/pym/portage/tests/repoman/test_simple.py b/pym/portage/tests/repoman/test_simple.py index 69eb36de8..5dbb7676f 100644 --- a/pym/portage/tests/repoman/test_simple.py +++ b/pym/portage/tests/repoman/test_simple.py @@ -9,7 +9,7 @@ import portage from portage import os from portage import shutil from portage import _unicode_decode -from portage.const import PORTAGE_BASE_PATH, PORTAGE_BIN_PATH, PORTAGE_PYM_PATH +from portage.const import PORTAGE_BASE_PATH, PORTAGE_PYM_PATH from portage.process import find_binary from portage.tests import TestCase from portage.tests.resolver.ResolverPlayground import ResolverPlayground @@ -171,7 +171,7 @@ class SimpleRepomanTestCase(TestCase): license_dir = os.path.join(test_repo_location, "licenses") repoman_cmd = (portage._python_interpreter, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "repoman")) + os.path.join(self.bindir, "repoman")) git_binary = find_binary("git") git_cmd = (git_binary,) @@ -274,7 +274,9 @@ class SimpleRepomanTestCase(TestCase): os.symlink(test_repo_location, test_repo_symlink) # repoman checks metadata.dtd for recent CTIME, so copy the file in # order to ensure that the CTIME is current - shutil.copyfile(metadata_dtd, os.path.join(distdir, "metadata.dtd")) + # NOTE: if we don't have the file around, let repoman try to fetch it. + if os.path.exists(metadata_dtd): + shutil.copyfile(metadata_dtd, os.path.join(distdir, "metadata.dtd")) if debug: # The subprocess inherits both stdout and stderr, for diff --git a/pym/portage/tests/resolver/ResolverPlayground.py b/pym/portage/tests/resolver/ResolverPlayground.py index 077e27159..2d162514c 100644 --- a/pym/portage/tests/resolver/ResolverPlayground.py +++ b/pym/portage/tests/resolver/ResolverPlayground.py @@ -15,6 +15,7 @@ from portage.package.ebuild.config import config from portage.package.ebuild.digestgen import digestgen from portage._sets import load_default_config from portage._sets.base import InternalPackageSet +from portage.tests import cnf_path from portage.util import ensure_dirs, normalize_path from portage.versions import catsplit @@ -58,15 +59,19 @@ class ResolverPlayground(object): def __init__(self, ebuilds={}, binpkgs={}, installed={}, profile={}, repo_configs={}, \ user_config={}, sets={}, world=[], world_sets=[], distfiles={}, - targetroot=False, debug=False): + eprefix=None, targetroot=False, debug=False): """ ebuilds: cpv -> metadata mapping simulating available ebuilds. installed: cpv -> metadata mapping simulating installed packages. If a metadata key is missing, it gets a default value. profile: settings defined by the profile. """ + self.debug = debug - self.eprefix = normalize_path(tempfile.mkdtemp()) + if eprefix is None: + self.eprefix = normalize_path(tempfile.mkdtemp()) + else: + self.eprefix = normalize_path(eprefix) portage.const.EPREFIX = self.eprefix.rstrip(os.sep) self.eroot = self.eprefix + os.sep @@ -414,7 +419,7 @@ class ResolverPlayground(object): make_globals_path = os.path.join(self.eroot, GLOBAL_CONFIG_PATH.lstrip(os.sep), "make.globals") ensure_dirs(os.path.dirname(make_globals_path)) - os.symlink(os.path.join(PORTAGE_BASE_PATH, "cnf", "make.globals"), + os.symlink(os.path.join(cnf_path, "make.globals"), make_globals_path) #Create /usr/share/portage/config/sets/portage.conf @@ -425,8 +430,8 @@ class ResolverPlayground(object): except os.error: pass - provided_sets_portage_conf = \ - os.path.join(PORTAGE_BASE_PATH, "cnf/sets/portage.conf") + provided_sets_portage_conf = ( + os.path.join(cnf_path, "sets", "portage.conf")) os.symlink(provided_sets_portage_conf, os.path.join(default_sets_conf_dir, "portage.conf")) set_config_dir = os.path.join(user_config_dir, "sets") @@ -544,6 +549,7 @@ class ResolverPlaygroundTestCase(object): self.all_permutations = kwargs.pop("all_permutations", False) self.ignore_mergelist_order = kwargs.pop("ignore_mergelist_order", False) self.ambiguous_merge_order = kwargs.pop("ambiguous_merge_order", False) + self.ambiguous_slot_collision_solutions = kwargs.pop("ambiguous_slot_collision_solutions", False) self.check_repo_names = kwargs.pop("check_repo_names", False) self.merge_order_assertions = kwargs.pop("merge_order_assertions", False) @@ -659,9 +665,22 @@ class ResolverPlaygroundTestCase(object): str((node1, node2))) + \ ", got: " + str(got)) - elif key in ("unstable_keywords", "needed_p_mask_changes") and expected is not None: + elif key == "slot_collision_solutions" and \ + self.ambiguous_slot_collision_solutions: + # Tests that use all_permutations can have multiple + # outcomes here. + for x in expected: + if x == got: + expected = x + break + elif key in ("unstable_keywords", "needed_p_mask_changes", + "unsatisfied_deps", "required_use_unsatisfied") and \ + expected is not None: expected = set(expected) + elif key == "forced_rebuilds" and expected is not None: + expected = dict((k, set(v)) for k, v in expected.items()) + if got != expected: fail_msgs.append("atoms: (" + ", ".join(result.atoms) + "), key: " + \ key + ", expected: " + str(expected) + ", got: " + str(got)) @@ -674,10 +693,15 @@ class ResolverPlaygroundTestCase(object): class ResolverPlaygroundResult(object): checks = ( - "success", "mergelist", "use_changes", "license_changes", "unstable_keywords", "slot_collision_solutions", + "success", "mergelist", "use_changes", "license_changes", + "unstable_keywords", "slot_collision_solutions", "circular_dependency_solutions", "needed_p_mask_changes", + "unsatisfied_deps", "forced_rebuilds", "required_use_unsatisfied" ) optional_checks = ( + "forced_rebuilds", + "required_use_unsatisfied", + "unsatisfied_deps" ) def __init__(self, atoms, success, mydepgraph, favorites): @@ -692,6 +716,9 @@ class ResolverPlaygroundResult(object): self.needed_p_mask_changes = None self.slot_collision_solutions = None self.circular_dependency_solutions = None + self.unsatisfied_deps = frozenset() + self.forced_rebuilds = None + self.required_use_unsatisfied = None if self.depgraph._dynamic_config._serialized_tasks_cache is not None: self.mergelist = [] @@ -751,6 +778,24 @@ class ResolverPlaygroundResult(object): sol = handler.solutions self.circular_dependency_solutions = dict(zip([x.cpv for x in sol.keys()], sol.values())) + if self.depgraph._dynamic_config._unsatisfied_deps_for_display: + self.unsatisfied_deps = set(dep_info[0][1] + for dep_info in self.depgraph._dynamic_config._unsatisfied_deps_for_display) + + if self.depgraph._forced_rebuilds: + self.forced_rebuilds = dict( + (child.cpv, set(parent.cpv for parent in parents)) + for child_dict in self.depgraph._forced_rebuilds.values() + for child, parents in child_dict.items()) + + required_use_unsatisfied = [] + for pargs, kwargs in \ + self.depgraph._dynamic_config._unsatisfied_deps_for_display: + if "show_req_use" in kwargs: + required_use_unsatisfied.append(pargs[1]) + if required_use_unsatisfied: + self.required_use_unsatisfied = set(required_use_unsatisfied) + class ResolverPlaygroundDepcleanResult(object): checks = ( diff --git a/pym/portage/tests/resolver/__test__ b/pym/portage/tests/resolver/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/resolver/__test__ +++ b/pym/portage/tests/resolver/__test__.py diff --git a/pym/portage/tests/resolver/test_autounmask_use_breakage.py b/pym/portage/tests/resolver/test_autounmask_use_breakage.py new file mode 100644 index 000000000..3654aa6a3 --- /dev/null +++ b/pym/portage/tests/resolver/test_autounmask_use_breakage.py @@ -0,0 +1,63 @@ +# Copyright 2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import (ResolverPlayground, + ResolverPlaygroundTestCase) + +class AutounmaskUseBreakageTestCase(TestCase): + + def testAutounmaskUseBreakage(self): + + ebuilds = { + + "app-misc/A-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/D[-foo]", + }, + + "app-misc/B-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/D[foo]" + }, + + "app-misc/C-0" : { + "EAPI": "5", + "RDEPEND": ">=app-misc/D-1" + }, + + "app-misc/D-0" : { + "EAPI": "5", + "IUSE": "foo" + }, + + "app-misc/D-1" : { + "EAPI": "5", + "IUSE": "bar" + }, + + } + + test_cases = ( + + # Bug 510270 + # _solve_non_slot_operator_slot_conflicts throws + # IndexError: tuple index out of range + # due to autounmask USE breakage. + ResolverPlaygroundTestCase( + ["app-misc/C", "app-misc/B", "app-misc/A"], + all_permutations = True, + success = False, + ambiguous_slot_collision_solutions = True, + slot_collision_solutions = [None, []] + ), + + ) + + playground = ResolverPlayground(ebuilds=ebuilds, debug=False) + try: + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, True, test_case.fail_msg) + finally: + playground.cleanup() diff --git a/pym/portage/tests/resolver/test_or_choices.py b/pym/portage/tests/resolver/test_or_choices.py index 90e681408..d9d14f058 100644 --- a/pym/portage/tests/resolver/test_or_choices.py +++ b/pym/portage/tests/resolver/test_or_choices.py @@ -132,3 +132,76 @@ class OrChoicesTestCase(TestCase): self.assertEqual(test_case.test_success, True, test_case.fail_msg) finally: playground.cleanup() + + + def testInitiallyUnsatisfied(self): + + ebuilds = { + + "app-misc/A-1" : { + "EAPI": "5", + "SLOT": "0/1" + }, + + "app-misc/A-2" : { + "EAPI": "5", + "SLOT": "0/2" + }, + + "app-misc/B-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/A:=" + }, + + "app-misc/C-0" : { + "EAPI": "5", + "RDEPEND": "|| ( app-misc/X <app-misc/A-2 )" + }, + + } + + installed = { + + "app-misc/A-1" : { + "EAPI": "5", + "SLOT": "0/1" + }, + + "app-misc/B-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/A:0/1=" + }, + + "app-misc/C-0" : { + "EAPI": "5", + "RDEPEND": "|| ( app-misc/X <app-misc/A-2 )" + }, + + } + + world = ["app-misc/B", "app-misc/C"] + + test_cases = ( + + # Test bug #522652, where the unsatisfiable app-misc/X + # atom is selected, and the dependency is placed into + # _initially_unsatisfied_deps where it is ignored, causing + # upgrade to app-misc/A-2 (breaking a dependency of + # app-misc/C-0). + ResolverPlaygroundTestCase( + ["app-misc/A"], + options = {}, + success = True, + mergelist = ['app-misc/A-1'] + ), + + ) + + playground = ResolverPlayground(ebuilds=ebuilds, + installed=installed, world=world, debug=False) + try: + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, True, test_case.fail_msg) + finally: + playground.cleanup() diff --git a/pym/portage/tests/resolver/test_slot_conflict_force_rebuild.py b/pym/portage/tests/resolver/test_slot_conflict_force_rebuild.py new file mode 100644 index 000000000..4170bfd9d --- /dev/null +++ b/pym/portage/tests/resolver/test_slot_conflict_force_rebuild.py @@ -0,0 +1,84 @@ +# Copyright 2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import (ResolverPlayground, + ResolverPlaygroundTestCase) + +class SlotConflictForceRebuildTestCase(TestCase): + + def testSlotConflictForceRebuild(self): + + ebuilds = { + + "app-misc/A-1" : { + "EAPI": "5", + "SLOT": "0/1" + }, + + "app-misc/A-2" : { + "EAPI": "5", + "SLOT": "0/2" + }, + + "app-misc/B-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/A:=" + }, + + "app-misc/C-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/A" + }, + + } + + installed = { + + "app-misc/A-1" : { + "EAPI": "5", + "SLOT": "0/1" + }, + + "app-misc/B-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/A:0/1=" + }, + + "app-misc/C-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/A:0/1=" + }, + + } + + world = ["app-misc/B", "app-misc/C"] + + test_cases = ( + + # Test bug #521990, where forced_rebuilds omits ebuilds that + # had have had their slot operator atoms removed from the + # ebuilds, even though the corresponding installed + # instances had really forced rebuilds due to being built + # with slot-operators in their deps. + ResolverPlaygroundTestCase( + ["app-misc/A"], + options = {}, + success = True, + ambiguous_merge_order = True, + mergelist = ['app-misc/A-2', ('app-misc/B-0', 'app-misc/C-0')], + forced_rebuilds = { + 'app-misc/A-2': ['app-misc/B-0', 'app-misc/C-0'] + } + ), + + ) + + playground = ResolverPlayground(ebuilds=ebuilds, + installed=installed, world=world, debug=False) + try: + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, True, test_case.fail_msg) + finally: + playground.cleanup() diff --git a/pym/portage/tests/resolver/test_slot_conflict_unsatisfied_deep_deps.py b/pym/portage/tests/resolver/test_slot_conflict_unsatisfied_deep_deps.py new file mode 100644 index 000000000..13f7e67e3 --- /dev/null +++ b/pym/portage/tests/resolver/test_slot_conflict_unsatisfied_deep_deps.py @@ -0,0 +1,115 @@ +# Copyright 2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import ResolverPlayground, ResolverPlaygroundTestCase + +class SlotConflictUnsatisfiedDeepDepsTestCase(TestCase): + + def testSlotConflictUnsatisfiedDeepDeps(self): + + ebuilds = { + "dev-libs/A-1": { }, + "dev-libs/A-2": { "KEYWORDS": "~x86" }, + "dev-libs/B-1": { "DEPEND": "dev-libs/A" }, + "dev-libs/C-1": { "DEPEND": ">=dev-libs/A-2" }, + "dev-libs/D-1": { "DEPEND": "dev-libs/A" }, + } + + installed = { + "dev-libs/broken-1": { + "RDEPEND": "dev-libs/A dev-libs/initially-unsatisfied" + }, + } + + world = ( + "dev-libs/A", + "dev-libs/B", + "dev-libs/C", + "dev-libs/D", + "dev-libs/broken" + ) + + test_cases = ( + # Test bug #520950, where unsatisfied deps of installed + # packages are supposed to be ignored when they are beyond + # the depth requested by the user. + ResolverPlaygroundTestCase( + ["dev-libs/B", "dev-libs/C", "dev-libs/D"], + all_permutations=True, + options={ + "--autounmask": "y", + "--complete-graph": True + }, + mergelist=["dev-libs/A-2", "dev-libs/B-1", "dev-libs/C-1", "dev-libs/D-1"], + ignore_mergelist_order=True, + unstable_keywords=["dev-libs/A-2"], + unsatisfied_deps=[], + success=False), + + ResolverPlaygroundTestCase( + ["@world"], + options={ + "--autounmask": "y", + "--complete-graph": True + }, + mergelist=["dev-libs/A-2", "dev-libs/B-1", "dev-libs/C-1", "dev-libs/D-1"], + ignore_mergelist_order=True, + unstable_keywords=["dev-libs/A-2"], + unsatisfied_deps=["dev-libs/broken"], + success=False), + + # Test --selective with --deep = 0 + ResolverPlaygroundTestCase( + ["@world"], + options={ + "--autounmask": "y", + "--complete-graph": True, + "--selective": True, + "--deep": 0 + }, + mergelist=["dev-libs/A-2", "dev-libs/B-1", "dev-libs/C-1", "dev-libs/D-1"], + ignore_mergelist_order=True, + unstable_keywords=["dev-libs/A-2"], + unsatisfied_deps=[], + success=False), + + # Test --deep = 1 + ResolverPlaygroundTestCase( + ["@world"], + options={ + "--autounmask": "y", + "--complete-graph": True, + "--selective": True, + "--deep": 1 + }, + mergelist=["dev-libs/A-2", "dev-libs/B-1", "dev-libs/C-1", "dev-libs/D-1"], + ignore_mergelist_order=True, + unstable_keywords=["dev-libs/A-2"], + unsatisfied_deps=["dev-libs/initially-unsatisfied"], + success=False), + + # Test --deep = True + ResolverPlaygroundTestCase( + ["@world"], + options={ + "--autounmask": "y", + "--complete-graph": True, + "--selective": True, + "--deep": True + }, + mergelist=["dev-libs/A-2", "dev-libs/B-1", "dev-libs/C-1", "dev-libs/D-1"], + ignore_mergelist_order=True, + unstable_keywords=["dev-libs/A-2"], + unsatisfied_deps=["dev-libs/initially-unsatisfied"], + success=False), + ) + + playground = ResolverPlayground(ebuilds=ebuilds, installed=installed, + world=world, debug=False) + try: + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, True, test_case.fail_msg) + finally: + playground.cleanup() diff --git a/pym/portage/tests/resolver/test_slot_operator_rebuild.py b/pym/portage/tests/resolver/test_slot_operator_rebuild.py new file mode 100644 index 000000000..42512aad8 --- /dev/null +++ b/pym/portage/tests/resolver/test_slot_operator_rebuild.py @@ -0,0 +1,80 @@ +# Copyright 2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import (ResolverPlayground, + ResolverPlaygroundTestCase) + +class SlotOperatorRebuildTestCase(TestCase): + + def testSlotOperatorRebuild(self): + + ebuilds = { + + "app-misc/A-1" : { + "EAPI": "5", + "SLOT": "0/1" + }, + + "app-misc/A-2" : { + "EAPI": "5", + "SLOT": "0/2" + }, + + "app-misc/B-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/A:=" + }, + + "app-misc/C-0" : { + "EAPI": "5", + "RDEPEND": "|| ( app-misc/X app-misc/A:= )" + }, + + } + + installed = { + + "app-misc/A-1" : { + "EAPI": "5", + "SLOT": "0/1" + }, + + "app-misc/B-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/A:0/1=" + }, + + "app-misc/C-0" : { + "EAPI": "5", + "RDEPEND": "|| ( app-misc/X app-misc/A:0/1= )" + }, + + } + + world = ["app-misc/B", "app-misc/C"] + + test_cases = ( + + # Test bug #522652, where the unsatisfiable app-misc/X + # atom is selected, and the dependency is placed into + # _initially_unsatisfied_deps where it is ignored, causing + # the app-misc/C-0 rebuild to be missed. + ResolverPlaygroundTestCase( + ["app-misc/A"], + options = {"--dynamic-deps": "n"}, + success = True, + ambiguous_merge_order = True, + mergelist = ['app-misc/A-2', ('app-misc/B-0', 'app-misc/C-0')] + ), + + ) + + playground = ResolverPlayground(ebuilds=ebuilds, + installed=installed, world=world, debug=False) + try: + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, True, test_case.fail_msg) + finally: + playground.cleanup() diff --git a/pym/portage/tests/resolver/test_slot_operator_required_use.py b/pym/portage/tests/resolver/test_slot_operator_required_use.py new file mode 100644 index 000000000..9cc6dbad4 --- /dev/null +++ b/pym/portage/tests/resolver/test_slot_operator_required_use.py @@ -0,0 +1,72 @@ +# Copyright 2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import (ResolverPlayground, + ResolverPlaygroundTestCase) + +class SlotOperatorRequiredUseTestCase(TestCase): + + def testSlotOperatorRequiredUse(self): + + ebuilds = { + + "app-misc/A-1" : { + "EAPI": "5", + "SLOT": "0/1" + }, + + "app-misc/A-2" : { + "EAPI": "5", + "SLOT": "0/2" + }, + + "app-misc/B-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/A:=", + "IUSE": "x y", + "REQUIRED_USE": "|| ( x y )" + }, + + } + + installed = { + + "app-misc/A-1" : { + "EAPI": "5", + "SLOT": "0/1" + }, + + "app-misc/B-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/A:0/1=", + "IUSE": "x y", + "USE": "x" + }, + + } + + world = ["app-misc/B"] + + test_cases = ( + + # bug 523048 + # Ensure that unsatisfied REQUIRED_USE is reported when + # it blocks necessary slot-operator rebuilds. + ResolverPlaygroundTestCase( + ["app-misc/A"], + success = False, + required_use_unsatisfied = ['app-misc/B:0'] + ), + + ) + + playground = ResolverPlayground(ebuilds=ebuilds, + installed=installed, world=world, debug=False) + try: + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, True, + test_case.fail_msg) + finally: + playground.cleanup() diff --git a/pym/portage/tests/resolver/test_solve_non_slot_operator_slot_conflicts.py b/pym/portage/tests/resolver/test_solve_non_slot_operator_slot_conflicts.py new file mode 100644 index 000000000..c6024f404 --- /dev/null +++ b/pym/portage/tests/resolver/test_solve_non_slot_operator_slot_conflicts.py @@ -0,0 +1,75 @@ +# Copyright 2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import (ResolverPlayground, + ResolverPlaygroundTestCase) + +class SolveNonSlotOperatorSlotConflictsTestCase(TestCase): + + def testSolveNonSlotOperatorSlotConflicts(self): + + ebuilds = { + + "app-misc/A-1" : { + "EAPI": "5", + "SLOT": "0/1", + "PDEPEND": "app-misc/B" + }, + + "app-misc/A-2" : { + "EAPI": "5", + "SLOT": "0/2", + "PDEPEND": "app-misc/B" + }, + + "app-misc/B-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/A:=" + }, + + } + + installed = { + + "app-misc/A-1" : { + "EAPI": "5", + "SLOT": "0/1", + "PDEPEND": "app-misc/B" + }, + + "app-misc/B-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/A:0/1=" + }, + + } + + world = ["app-misc/A"] + + test_cases = ( + + # bug 522084 + # In this case, _solve_non_slot_operator_slot_conflicts + # removed both versions of app-misc/A from the graph, since + # they didn't have any non-conflict parents (except for + # @selected which matched both instances). The result was + # a missed update. + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True}, + success = True, + mergelist = ['app-misc/A-2', 'app-misc/B-0'] + ), + + ) + + playground = ResolverPlayground(ebuilds=ebuilds, + installed=installed, world=world, debug=False) + try: + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, True, + test_case.fail_msg) + finally: + playground.cleanup() diff --git a/pym/portage/tests/runTests b/pym/portage/tests/runTests.py index a80668b8d..a80668b8d 100755 --- a/pym/portage/tests/runTests +++ b/pym/portage/tests/runTests.py diff --git a/pym/portage/tests/sets/base/__test__ b/pym/portage/tests/sets/base/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/sets/base/__test__ +++ b/pym/portage/tests/sets/base/__test__.py diff --git a/pym/portage/tests/sets/files/__test__ b/pym/portage/tests/sets/files/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/sets/files/__test__ +++ b/pym/portage/tests/sets/files/__test__.py diff --git a/pym/portage/tests/sets/shell/__test__ b/pym/portage/tests/sets/shell/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/sets/shell/__test__ +++ b/pym/portage/tests/sets/shell/__test__.py diff --git a/pym/portage/tests/unicode/__test__ b/pym/portage/tests/unicode/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/unicode/__test__ +++ b/pym/portage/tests/unicode/__test__.py diff --git a/pym/portage/tests/update/__test__ b/pym/portage/tests/update/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/update/__test__ +++ b/pym/portage/tests/update/__test__.py diff --git a/pym/portage/tests/util/__test__ b/pym/portage/tests/util/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/util/__test__ +++ b/pym/portage/tests/util/__test__.py diff --git a/pym/portage/tests/util/test_getconfig.py b/pym/portage/tests/util/test_getconfig.py index e5fd60f6d..b72bd6a02 100644 --- a/pym/portage/tests/util/test_getconfig.py +++ b/pym/portage/tests/util/test_getconfig.py @@ -26,9 +26,7 @@ class GetConfigTestCase(TestCase): } def testGetConfig(self): - - make_globals_file = os.path.join(PORTAGE_BASE_PATH, - 'cnf', 'make.globals') + make_globals_file = os.path.join(self.cnf_path, "make.globals") d = getconfig(make_globals_file) for k, v in self._cases.items(): self.assertEqual(d[k], v) diff --git a/pym/portage/tests/versions/__test__ b/pym/portage/tests/versions/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/versions/__test__ +++ b/pym/portage/tests/versions/__test__.py diff --git a/pym/portage/tests/xpak/__test__ b/pym/portage/tests/xpak/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/xpak/__test__ +++ b/pym/portage/tests/xpak/__test__.py diff --git a/pym/portage/util/__init__.py b/pym/portage/util/__init__.py index 9e2d1e40e..d48b47a40 100644 --- a/pym/portage/util/__init__.py +++ b/pym/portage/util/__init__.py @@ -43,7 +43,8 @@ from portage import _unicode_encode from portage import _unicode_decode from portage.const import VCS_DIRS from portage.exception import InvalidAtom, PortageException, FileNotFound, \ - OperationNotPermitted, ParseError, PermissionDenied, ReadOnlyFileSystem + IsADirectory, OperationNotPermitted, ParseError, PermissionDenied, \ + ReadOnlyFileSystem from portage.localization import _ from portage.proxy.objectproxy import ObjectProxy from portage.cache.mappings import UserDict diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py index 9ffcc74d9..809540023 100644 --- a/pym/portage/util/_eventloop/EventLoop.py +++ b/pym/portage/util/_eventloop/EventLoop.py @@ -1,6 +1,8 @@ -# Copyright 1999-2013 Gentoo Foundation +# Copyright 1999-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import division + import errno import logging import os @@ -211,7 +213,7 @@ class EventLoop(object): if timeout is None: wait_timeout = None else: - wait_timeout = float(timeout) / 1000 + wait_timeout = timeout / 1000 # NOTE: In order to avoid a possible infinite wait when # wait_timeout is None, the previous _run_timeouts() # call must have returned False *with* _thread_condition @@ -657,6 +659,6 @@ class _epoll_adapter(object): if timeout is None or timeout < 0: timeout = -1 elif timeout != 0: - timeout = float(timeout) / 1000 + timeout = timeout / 1000 return self._epoll_obj.poll(timeout) diff --git a/pym/portage/util/_eventloop/PollSelectAdapter.py b/pym/portage/util/_eventloop/PollSelectAdapter.py index 244788c57..32b404b67 100644 --- a/pym/portage/util/_eventloop/PollSelectAdapter.py +++ b/pym/portage/util/_eventloop/PollSelectAdapter.py @@ -1,6 +1,8 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import division + from .PollConstants import PollConstants import select @@ -64,7 +66,7 @@ class PollSelectAdapter(object): if timeout is not None and timeout < 0: timeout = None if timeout is not None: - select_args.append(float(timeout) / 1000) + select_args.append(timeout / 1000) select_events = select.select(*select_args) poll_events = [] |