From bc0fa8d3795ed7e40aaa00f579bb2977897bce25 Mon Sep 17 00:00:00 2001 From: Michał Górny Date: Tue, 17 Jul 2018 21:50:45 +0200 Subject: Rename pym→lib, for better distutils-r1 interoperability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes: https://github.com/gentoo/portage/pull/343 --- pym/portage/dbapi/vartree.py | 5559 ------------------------------------------ 1 file changed, 5559 deletions(-) delete mode 100644 pym/portage/dbapi/vartree.py (limited to 'pym/portage/dbapi/vartree.py') diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py deleted file mode 100644 index a104306eb..000000000 --- a/pym/portage/dbapi/vartree.py +++ /dev/null @@ -1,5559 +0,0 @@ -# Copyright 1998-2018 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from __future__ import division, unicode_literals - -__all__ = [ - "vardbapi", "vartree", "dblink"] + \ - ["write_contents", "tar_contents"] - -import portage -portage.proxy.lazyimport.lazyimport(globals(), - 'hashlib:md5', - 'portage.checksum:_perform_md5_merge@perform_md5', - 'portage.data:portage_gid,portage_uid,secpass', - 'portage.dbapi.dep_expand:dep_expand', - 'portage.dbapi._MergeProcess:MergeProcess', - 'portage.dbapi._SyncfsProcess:SyncfsProcess', - 'portage.dep:dep_getkey,isjustname,isvalidatom,match_from_list,' + \ - 'use_reduce,_slot_separator,_repo_separator', - 'portage.eapi:_get_eapi_attrs', - 'portage.elog:collect_ebuild_messages,collect_messages,' + \ - 'elog_process,_merge_logentries', - 'portage.locks:lockdir,unlockdir,lockfile,unlockfile', - 'portage.output:bold,colorize', - 'portage.package.ebuild.doebuild:doebuild_environment,' + \ - '_merge_unicode_error', '_spawn_phase', - 'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs', - 'portage.package.ebuild._ipc.QueryCommand:QueryCommand', - 'portage.process:find_binary', - 'portage.util:apply_secpass_permissions,ConfigProtect,ensure_dirs,' + \ - 'writemsg,writemsg_level,write_atomic,atomic_ofstream,writedict,' + \ - 'grabdict,normalize_path,new_protect_filename', - 'portage.util.digraph:digraph', - 'portage.util.env_update:env_update', - 'portage.util.install_mask:install_mask_dir,InstallMask', - 'portage.util.listdir:dircache,listdir', - 'portage.util.movefile:movefile', - 'portage.util.path:first_existing,iter_parents', - 'portage.util.writeable_check:get_ro_checker', - 'portage.util._xattr:xattr', - 'portage.util._dyn_libs.PreservedLibsRegistry:PreservedLibsRegistry', - 'portage.util._dyn_libs.LinkageMapELF:LinkageMapELF@LinkageMap', - 'portage.util._dyn_libs.NeededEntry:NeededEntry', - 'portage.util._async.SchedulerInterface:SchedulerInterface', - 'portage.util._eventloop.EventLoop:EventLoop', - 'portage.util._eventloop.global_event_loop:global_event_loop', - 'portage.versions:best,catpkgsplit,catsplit,cpv_getkey,vercmp,' + \ - '_get_slot_re,_pkgsplit@pkgsplit,_pkg_str,_unknown_repo', - 'subprocess', - 'tarfile', -) - -from portage.const import CACHE_PATH, CONFIG_MEMORY_FILE, \ - MERGING_IDENTIFIER, PORTAGE_PACKAGE_ATOM, PRIVATE_PATH, VDB_PATH -from portage.dbapi import dbapi -from portage.exception import CommandNotFound, \ - InvalidData, InvalidLocation, InvalidPackageName, \ - FileNotFound, PermissionDenied, UnsupportedAPIException -from portage.localization import _ - -from portage import abssymlink, _movefile, bsd_chflags - -# This is a special version of the os module, wrapped for unicode support. -from portage import os -from portage import shutil -from portage import _encodings -from portage import _os_merge -from portage import _selinux_merge -from portage import _unicode_decode -from portage import _unicode_encode -from ._VdbMetadataDelta import VdbMetadataDelta - -from _emerge.EbuildBuildDir import EbuildBuildDir -from _emerge.EbuildPhase import EbuildPhase -from _emerge.emergelog import emergelog -from _emerge.MiscFunctionsProcess import MiscFunctionsProcess -from _emerge.SpawnProcess import SpawnProcess -from ._ContentsCaseSensitivityManager import ContentsCaseSensitivityManager - -import errno -import fnmatch -import gc -import grp -import io -from itertools import chain -import logging -import os as _os -import platform -import pwd -import re -import stat -import sys -import tempfile -import textwrap -import time -import warnings - -try: - import cPickle as pickle -except ImportError: - import pickle - -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str - long = int - _unicode = str -else: - _unicode = unicode - -class vardbapi(dbapi): - - _excluded_dirs = ["CVS", "lost+found"] - _excluded_dirs = [re.escape(x) for x in _excluded_dirs] - _excluded_dirs = re.compile(r'^(\..*|' + MERGING_IDENTIFIER + '.*|' + \ - "|".join(_excluded_dirs) + r')$') - - _aux_cache_version = "1" - _owners_cache_version = "1" - - # Number of uncached packages to trigger cache update, since - # it's wasteful to update it for every vdb change. - _aux_cache_threshold = 5 - - _aux_cache_keys_re = re.compile(r'^NEEDED\..*$') - _aux_multi_line_re = re.compile(r'^(CONTENTS|NEEDED\..*)$') - - def __init__(self, _unused_param=DeprecationWarning, - categories=None, settings=None, vartree=None): - """ - The categories parameter is unused since the dbapi class - now has a categories property that is generated from the - available packages. - """ - - # Used by emerge to check whether any packages - # have been added or removed. - self._pkgs_changed = False - - # The _aux_cache_threshold doesn't work as designed - # if the cache is flushed from a subprocess, so we - # use this to avoid waste vdb cache updates. - self._flush_cache_enabled = True - - #cache for category directory mtimes - self.mtdircache = {} - - #cache for dependency checks - self.matchcache = {} - - #cache for cp_list results - self.cpcache = {} - - self.blockers = None - if settings is None: - settings = portage.settings - self.settings = settings - - if _unused_param is not DeprecationWarning: - warnings.warn("The first parameter of the " - "portage.dbapi.vartree.vardbapi" - " constructor is now unused. Instead " - "settings['ROOT'] is used.", - DeprecationWarning, stacklevel=2) - - self._eroot = settings['EROOT'] - self._dbroot = self._eroot + VDB_PATH - self._lock = None - self._lock_count = 0 - - self._conf_mem_file = self._eroot + CONFIG_MEMORY_FILE - self._fs_lock_obj = None - self._fs_lock_count = 0 - self._slot_locks = {} - - if vartree is None: - vartree = portage.db[settings['EROOT']]['vartree'] - self.vartree = vartree - self._aux_cache_keys = set( - ["BDEPEND", "BUILD_TIME", "CHOST", "COUNTER", "DEPEND", - "DESCRIPTION", "EAPI", "HDEPEND", "HOMEPAGE", - "BUILD_ID", "IUSE", "KEYWORDS", - "LICENSE", "PDEPEND", "PROPERTIES", "RDEPEND", - "repository", "RESTRICT" , "SLOT", "USE", "DEFINED_PHASES", - "PROVIDES", "REQUIRES" - ]) - self._aux_cache_obj = None - self._aux_cache_filename = os.path.join(self._eroot, - CACHE_PATH, "vdb_metadata.pickle") - self._cache_delta_filename = os.path.join(self._eroot, - CACHE_PATH, "vdb_metadata_delta.json") - self._cache_delta = VdbMetadataDelta(self) - self._counter_path = os.path.join(self._eroot, - CACHE_PATH, "counter") - - self._plib_registry = PreservedLibsRegistry(settings["ROOT"], - os.path.join(self._eroot, PRIVATE_PATH, "preserved_libs_registry")) - self._linkmap = LinkageMap(self) - self._owners = self._owners_db(self) - - self._cached_counter = None - - @property - def writable(self): - """ - Check if var/db/pkg is writable, or permissions are sufficient - to create it if it does not exist yet. - @rtype: bool - @return: True if var/db/pkg is writable or can be created, - False otherwise - """ - return os.access(first_existing(self._dbroot), os.W_OK) - - @property - def root(self): - warnings.warn("The root attribute of " - "portage.dbapi.vartree.vardbapi" - " is deprecated. Use " - "settings['ROOT'] instead.", - DeprecationWarning, stacklevel=3) - return self.settings['ROOT'] - - def getpath(self, mykey, filename=None): - # This is an optimized hotspot, so don't use unicode-wrapped - # os module and don't use os.path.join(). - rValue = self._eroot + VDB_PATH + _os.sep + mykey - if filename is not None: - # If filename is always relative, we can do just - # rValue += _os.sep + filename - rValue = _os.path.join(rValue, filename) - return rValue - - def lock(self): - """ - Acquire a reentrant lock, blocking, for cooperation with concurrent - processes. State is inherited by subprocesses, allowing subprocesses - to reenter a lock that was acquired by a parent process. However, - a lock can be released only by the same process that acquired it. - """ - if self._lock_count: - self._lock_count += 1 - else: - if self._lock is not None: - raise AssertionError("already locked") - # At least the parent needs to exist for the lock file. - ensure_dirs(self._dbroot) - self._lock = lockdir(self._dbroot) - self._lock_count += 1 - - def unlock(self): - """ - Release a lock, decrementing the recursion level. Each unlock() call - must be matched with a prior lock() call, or else an AssertionError - will be raised if unlock() is called while not locked. - """ - if self._lock_count > 1: - self._lock_count -= 1 - else: - if self._lock is None: - raise AssertionError("not locked") - self._lock_count = 0 - unlockdir(self._lock) - self._lock = None - - def _fs_lock(self): - """ - Acquire a reentrant lock, blocking, for cooperation with concurrent - processes. - """ - if self._fs_lock_count < 1: - if self._fs_lock_obj is not None: - raise AssertionError("already locked") - try: - self._fs_lock_obj = lockfile(self._conf_mem_file) - except InvalidLocation: - self.settings._init_dirs() - self._fs_lock_obj = lockfile(self._conf_mem_file) - self._fs_lock_count += 1 - - def _fs_unlock(self): - """ - Release a lock, decrementing the recursion level. - """ - if self._fs_lock_count <= 1: - if self._fs_lock_obj is None: - raise AssertionError("not locked") - unlockfile(self._fs_lock_obj) - self._fs_lock_obj = None - self._fs_lock_count -= 1 - - def _slot_lock(self, slot_atom): - """ - Acquire a slot lock (reentrant). - - WARNING: The varbapi._slot_lock method is not safe to call - in the main process when that process is scheduling - install/uninstall tasks in parallel, since the locks would - be inherited by child processes. In order to avoid this sort - of problem, this method should be called in a subprocess - (typically spawned by the MergeProcess class). - """ - lock, counter = self._slot_locks.get(slot_atom, (None, 0)) - if lock is None: - lock_path = self.getpath("%s:%s" % (slot_atom.cp, slot_atom.slot)) - ensure_dirs(os.path.dirname(lock_path)) - lock = lockfile(lock_path, wantnewlockfile=True) - self._slot_locks[slot_atom] = (lock, counter + 1) - - def _slot_unlock(self, slot_atom): - """ - Release a slot lock (or decrementing recursion level). - """ - lock, counter = self._slot_locks.get(slot_atom, (None, 0)) - if lock is None: - raise AssertionError("not locked") - counter -= 1 - if counter == 0: - unlockfile(lock) - del self._slot_locks[slot_atom] - else: - self._slot_locks[slot_atom] = (lock, counter) - - def _bump_mtime(self, cpv): - """ - This is called before an after any modifications, so that consumers - can use directory mtimes to validate caches. See bug #290428. - """ - base = self._eroot + VDB_PATH - cat = catsplit(cpv)[0] - catdir = base + _os.sep + cat - t = time.time() - t = (t, t) - try: - for x in (catdir, base): - os.utime(x, t) - except OSError: - ensure_dirs(catdir) - - def cpv_exists(self, mykey, myrepo=None): - "Tells us whether an actual ebuild exists on disk (no masking)" - return os.path.exists(self.getpath(mykey)) - - def cpv_counter(self, mycpv): - "This method will grab the COUNTER. Returns a counter value." - try: - return long(self.aux_get(mycpv, ["COUNTER"])[0]) - except (KeyError, ValueError): - pass - writemsg_level(_("portage: COUNTER for %s was corrupted; " \ - "resetting to value of 0\n") % (mycpv,), - level=logging.ERROR, noiselevel=-1) - return 0 - - def cpv_inject(self, mycpv): - "injects a real package into our on-disk database; assumes mycpv is valid and doesn't already exist" - ensure_dirs(self.getpath(mycpv)) - counter = self.counter_tick(mycpv=mycpv) - # write local package counter so that emerge clean does the right thing - write_atomic(self.getpath(mycpv, filename="COUNTER"), str(counter)) - - def isInjected(self, mycpv): - if self.cpv_exists(mycpv): - if os.path.exists(self.getpath(mycpv, filename="INJECTED")): - return True - if not os.path.exists(self.getpath(mycpv, filename="CONTENTS")): - return True - return False - - def move_ent(self, mylist, repo_match=None): - origcp = mylist[1] - newcp = mylist[2] - - # sanity check - for atom in (origcp, newcp): - if not isjustname(atom): - raise InvalidPackageName(str(atom)) - origmatches = self.match(origcp, use_cache=0) - moves = 0 - if not origmatches: - return moves - for mycpv in origmatches: - try: - mycpv = self._pkg_str(mycpv, None) - except (KeyError, InvalidData): - continue - mycpv_cp = cpv_getkey(mycpv) - if mycpv_cp != origcp: - # Ignore PROVIDE virtual match. - continue - if repo_match is not None \ - and not repo_match(mycpv.repo): - continue - - # Use isvalidatom() to check if this move is valid for the - # EAPI (characters allowed in package names may vary). - if not isvalidatom(newcp, eapi=mycpv.eapi): - continue - - mynewcpv = mycpv.replace(mycpv_cp, _unicode(newcp), 1) - mynewcat = catsplit(newcp)[0] - origpath = self.getpath(mycpv) - if not os.path.exists(origpath): - continue - moves += 1 - if not os.path.exists(self.getpath(mynewcat)): - #create the directory - ensure_dirs(self.getpath(mynewcat)) - newpath = self.getpath(mynewcpv) - if os.path.exists(newpath): - #dest already exists; keep this puppy where it is. - continue - _movefile(origpath, newpath, mysettings=self.settings) - self._clear_pkg_cache(self._dblink(mycpv)) - self._clear_pkg_cache(self._dblink(mynewcpv)) - - # We need to rename the ebuild now. - old_pf = catsplit(mycpv)[1] - new_pf = catsplit(mynewcpv)[1] - if new_pf != old_pf: - try: - os.rename(os.path.join(newpath, old_pf + ".ebuild"), - os.path.join(newpath, new_pf + ".ebuild")) - except EnvironmentError as e: - if e.errno != errno.ENOENT: - raise - del e - write_atomic(os.path.join(newpath, "PF"), new_pf+"\n") - write_atomic(os.path.join(newpath, "CATEGORY"), mynewcat+"\n") - - return moves - - def cp_list(self, mycp, use_cache=1): - mysplit=catsplit(mycp) - if mysplit[0] == '*': - mysplit[0] = mysplit[0][1:] - try: - if sys.hexversion >= 0x3030000: - mystat = os.stat(self.getpath(mysplit[0])).st_mtime_ns - else: - mystat = os.stat(self.getpath(mysplit[0])).st_mtime - except OSError: - mystat = 0 - if use_cache and mycp in self.cpcache: - cpc = self.cpcache[mycp] - if cpc[0] == mystat: - return cpc[1][:] - cat_dir = self.getpath(mysplit[0]) - try: - dir_list = os.listdir(cat_dir) - except EnvironmentError as e: - if e.errno == PermissionDenied.errno: - raise PermissionDenied(cat_dir) - del e - dir_list = [] - - returnme = [] - for x in dir_list: - if self._excluded_dirs.match(x) is not None: - continue - ps = pkgsplit(x) - if not ps: - self.invalidentry(os.path.join(self.getpath(mysplit[0]), x)) - continue - if len(mysplit) > 1: - if ps[0] == mysplit[1]: - cpv = "%s/%s" % (mysplit[0], x) - metadata = dict(zip(self._aux_cache_keys, - self.aux_get(cpv, self._aux_cache_keys))) - returnme.append(_pkg_str(cpv, metadata=metadata, - settings=self.settings, db=self)) - self._cpv_sort_ascending(returnme) - if use_cache: - self.cpcache[mycp] = [mystat, returnme[:]] - elif mycp in self.cpcache: - del self.cpcache[mycp] - return returnme - - def cpv_all(self, use_cache=1): - """ - Set use_cache=0 to bypass the portage.cachedir() cache in cases - when the accuracy of mtime staleness checks should not be trusted - (generally this is only necessary in critical sections that - involve merge or unmerge of packages). - """ - return list(self._iter_cpv_all(use_cache=use_cache)) - - def _iter_cpv_all(self, use_cache=True, sort=False): - returnme = [] - basepath = os.path.join(self._eroot, VDB_PATH) + os.path.sep - - if use_cache: - from portage import listdir - else: - def listdir(p, **kwargs): - try: - return [x for x in os.listdir(p) \ - if os.path.isdir(os.path.join(p, x))] - except EnvironmentError as e: - if e.errno == PermissionDenied.errno: - raise PermissionDenied(p) - del e - return [] - - catdirs = listdir(basepath, EmptyOnError=1, ignorecvs=1, dirsonly=1) - if sort: - catdirs.sort() - - for x in catdirs: - if self._excluded_dirs.match(x) is not None: - continue - if not self._category_re.match(x): - continue - - pkgdirs = listdir(basepath + x, EmptyOnError=1, dirsonly=1) - if sort: - pkgdirs.sort() - - for y in pkgdirs: - if self._excluded_dirs.match(y) is not None: - continue - subpath = x + "/" + y - # -MERGING- should never be a cpv, nor should files. - try: - subpath = _pkg_str(subpath, db=self) - except InvalidData: - self.invalidentry(self.getpath(subpath)) - continue - - yield subpath - - def cp_all(self, use_cache=1, sort=False): - mylist = self.cpv_all(use_cache=use_cache) - d={} - for y in mylist: - if y[0] == '*': - y = y[1:] - try: - mysplit = catpkgsplit(y) - except InvalidData: - self.invalidentry(self.getpath(y)) - continue - if not mysplit: - self.invalidentry(self.getpath(y)) - continue - d[mysplit[0]+"/"+mysplit[1]] = None - return sorted(d) if sort else list(d) - - def checkblockers(self, origdep): - pass - - def _clear_cache(self): - self.mtdircache.clear() - self.matchcache.clear() - self.cpcache.clear() - self._aux_cache_obj = None - - def _add(self, pkg_dblink): - self._pkgs_changed = True - self._clear_pkg_cache(pkg_dblink) - - def _remove(self, pkg_dblink): - self._pkgs_changed = True - self._clear_pkg_cache(pkg_dblink) - - def _clear_pkg_cache(self, pkg_dblink): - # Due to 1 second mtime granularity in = 0x3030000: - curmtime = os.stat(os.path.join(self._eroot, VDB_PATH, mycat)).st_mtime_ns - else: - curmtime = os.stat(os.path.join(self._eroot, VDB_PATH, mycat)).st_mtime - except (IOError, OSError): - curmtime=0 - - if mycat not in self.matchcache or \ - self.mtdircache[mycat] != curmtime: - # clear cache entry - self.mtdircache[mycat] = curmtime - self.matchcache[mycat] = {} - if mydep not in self.matchcache[mycat]: - mymatch = list(self._iter_match(mydep, - self.cp_list(mydep.cp, use_cache=use_cache))) - self.matchcache[mycat][cache_key] = mymatch - return self.matchcache[mycat][cache_key][:] - - def findname(self, mycpv, myrepo=None): - return self.getpath(str(mycpv), filename=catsplit(mycpv)[1]+".ebuild") - - def flush_cache(self): - """If the current user has permission and the internal aux_get cache has - been updated, save it to disk and mark it unmodified. This is called - by emerge after it has loaded the full vdb for use in dependency - calculations. Currently, the cache is only written if the user has - superuser privileges (since that's required to obtain a lock), but all - users have read access and benefit from faster metadata lookups (as - long as at least part of the cache is still valid).""" - if self._flush_cache_enabled and \ - self._aux_cache is not None and \ - secpass >= 2 and \ - (len(self._aux_cache["modified"]) >= self._aux_cache_threshold or - not os.path.exists(self._cache_delta_filename)): - - ensure_dirs(os.path.dirname(self._aux_cache_filename)) - - self._owners.populate() # index any unindexed contents - valid_nodes = set(self.cpv_all()) - for cpv in list(self._aux_cache["packages"]): - if cpv not in valid_nodes: - del self._aux_cache["packages"][cpv] - del self._aux_cache["modified"] - timestamp = time.time() - self._aux_cache["timestamp"] = timestamp - - f = atomic_ofstream(self._aux_cache_filename, 'wb') - pickle.dump(self._aux_cache, f, protocol=2) - f.close() - apply_secpass_permissions( - self._aux_cache_filename, mode=0o644) - - self._cache_delta.initialize(timestamp) - apply_secpass_permissions( - self._cache_delta_filename, mode=0o644) - - self._aux_cache["modified"] = set() - - @property - def _aux_cache(self): - if self._aux_cache_obj is None: - self._aux_cache_init() - return self._aux_cache_obj - - def _aux_cache_init(self): - aux_cache = None - open_kwargs = {} - if sys.hexversion >= 0x3000000 and sys.hexversion < 0x3020000: - # Buffered io triggers extreme performance issues in - # Unpickler.load() (problem observed with python-3.0.1). - # Unfortunately, performance is still poor relative to - # python-2.x, but buffering makes it much worse (problem - # appears to be solved in Python >=3.2 at least). - open_kwargs["buffering"] = 0 - try: - with open(_unicode_encode(self._aux_cache_filename, - encoding=_encodings['fs'], errors='strict'), - mode='rb', **open_kwargs) as f: - mypickle = pickle.Unpickler(f) - try: - mypickle.find_global = None - except AttributeError: - # TODO: If py3k, override Unpickler.find_class(). - pass - aux_cache = mypickle.load() - except (SystemExit, KeyboardInterrupt): - raise - except Exception as e: - if isinstance(e, EnvironmentError) and \ - getattr(e, 'errno', None) in (errno.ENOENT, errno.EACCES): - pass - else: - writemsg(_("!!! Error loading '%s': %s\n") % \ - (self._aux_cache_filename, e), noiselevel=-1) - del e - - if not aux_cache or \ - not isinstance(aux_cache, dict) or \ - aux_cache.get("version") != self._aux_cache_version or \ - not aux_cache.get("packages"): - aux_cache = {"version": self._aux_cache_version} - aux_cache["packages"] = {} - - owners = aux_cache.get("owners") - if owners is not None: - if not isinstance(owners, dict): - owners = None - elif "version" not in owners: - owners = None - elif owners["version"] != self._owners_cache_version: - owners = None - elif "base_names" not in owners: - owners = None - elif not isinstance(owners["base_names"], dict): - owners = None - - if owners is None: - owners = { - "base_names" : {}, - "version" : self._owners_cache_version - } - aux_cache["owners"] = owners - - aux_cache["modified"] = set() - self._aux_cache_obj = aux_cache - - def aux_get(self, mycpv, wants, myrepo = None): - """This automatically caches selected keys that are frequently needed - by emerge for dependency calculations. The cached metadata is - considered valid if the mtime of the package directory has not changed - since the data was cached. The cache is stored in a pickled dict - object with the following format: - - {version:"1", "packages":{cpv1:(mtime,{k1,v1, k2,v2, ...}), cpv2...}} - - If an error occurs while loading the cache pickle or the version is - unrecognized, the cache will simple be recreated from scratch (it is - completely disposable). - """ - cache_these_wants = self._aux_cache_keys.intersection(wants) - for x in wants: - if self._aux_cache_keys_re.match(x) is not None: - cache_these_wants.add(x) - - if not cache_these_wants: - mydata = self._aux_get(mycpv, wants) - return [mydata[x] for x in wants] - - cache_these = set(self._aux_cache_keys) - cache_these.update(cache_these_wants) - - mydir = self.getpath(mycpv) - mydir_stat = None - try: - mydir_stat = os.stat(mydir) - except OSError as e: - if e.errno != errno.ENOENT: - raise - raise KeyError(mycpv) - # Use float mtime when available. - mydir_mtime = mydir_stat.st_mtime - pkg_data = self._aux_cache["packages"].get(mycpv) - pull_me = cache_these.union(wants) - mydata = {"_mtime_" : mydir_mtime} - cache_valid = False - cache_incomplete = False - cache_mtime = None - metadata = None - if pkg_data is not None: - if not isinstance(pkg_data, tuple) or len(pkg_data) != 2: - pkg_data = None - else: - cache_mtime, metadata = pkg_data - if not isinstance(cache_mtime, (float, long, int)) or \ - not isinstance(metadata, dict): - pkg_data = None - - if pkg_data: - cache_mtime, metadata = pkg_data - if isinstance(cache_mtime, float): - if cache_mtime == mydir_stat.st_mtime: - cache_valid = True - - # Handle truncated mtime in order to avoid cache - # invalidation for livecd squashfs (bug 564222). - elif long(cache_mtime) == mydir_stat.st_mtime: - cache_valid = True - else: - # Cache may contain integer mtime. - cache_valid = cache_mtime == mydir_stat[stat.ST_MTIME] - - if cache_valid: - # Migrate old metadata to unicode. - for k, v in metadata.items(): - metadata[k] = _unicode_decode(v, - encoding=_encodings['repo.content'], errors='replace') - - mydata.update(metadata) - pull_me.difference_update(mydata) - - if pull_me: - # pull any needed data and cache it - aux_keys = list(pull_me) - mydata.update(self._aux_get(mycpv, aux_keys, st=mydir_stat)) - if not cache_valid or cache_these.difference(metadata): - cache_data = {} - if cache_valid and metadata: - cache_data.update(metadata) - for aux_key in cache_these: - cache_data[aux_key] = mydata[aux_key] - self._aux_cache["packages"][_unicode(mycpv)] = \ - (mydir_mtime, cache_data) - self._aux_cache["modified"].add(mycpv) - - eapi_attrs = _get_eapi_attrs(mydata['EAPI']) - if _get_slot_re(eapi_attrs).match(mydata['SLOT']) is None: - # Empty or invalid slot triggers InvalidAtom exceptions when - # generating slot atoms for packages, so translate it to '0' here. - mydata['SLOT'] = '0' - - return [mydata[x] for x in wants] - - def _aux_get(self, mycpv, wants, st=None): - mydir = self.getpath(mycpv) - if st is None: - try: - st = os.stat(mydir) - except OSError as e: - if e.errno == errno.ENOENT: - raise KeyError(mycpv) - elif e.errno == PermissionDenied.errno: - raise PermissionDenied(mydir) - else: - raise - if not stat.S_ISDIR(st.st_mode): - raise KeyError(mycpv) - results = {} - env_keys = [] - for x in wants: - if x == "_mtime_": - results[x] = st[stat.ST_MTIME] - continue - try: - with io.open( - _unicode_encode(os.path.join(mydir, x), - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], - errors='replace') as f: - myd = f.read() - except IOError: - if x not in self._aux_cache_keys and \ - self._aux_cache_keys_re.match(x) is None: - env_keys.append(x) - continue - myd = '' - - # Preserve \n for metadata that is known to - # contain multiple lines. - if self._aux_multi_line_re.match(x) is None: - myd = " ".join(myd.split()) - - results[x] = myd - - if env_keys: - env_results = self._aux_env_search(mycpv, env_keys) - for k in env_keys: - v = env_results.get(k) - if v is None: - v = '' - if self._aux_multi_line_re.match(k) is None: - v = " ".join(v.split()) - results[k] = v - - if results.get("EAPI") == "": - results["EAPI"] = '0' - - return results - - def _aux_env_search(self, cpv, variables): - """ - Search environment.bz2 for the specified variables. Returns - a dict mapping variables to values, and any variables not - found in the environment will not be included in the dict. - This is useful for querying variables like ${SRC_URI} and - ${A}, which are not saved in separate files but are available - in environment.bz2 (see bug #395463). - """ - env_file = self.getpath(cpv, filename="environment.bz2") - if not os.path.isfile(env_file): - return {} - bunzip2_cmd = portage.util.shlex_split( - self.settings.get("PORTAGE_BUNZIP2_COMMAND", "")) - if not bunzip2_cmd: - bunzip2_cmd = portage.util.shlex_split( - self.settings["PORTAGE_BZIP2_COMMAND"]) - bunzip2_cmd.append("-d") - args = bunzip2_cmd + ["-c", env_file] - try: - proc = subprocess.Popen(args, stdout=subprocess.PIPE) - except EnvironmentError as e: - if e.errno != errno.ENOENT: - raise - raise portage.exception.CommandNotFound(args[0]) - - # Parts of the following code are borrowed from - # filter-bash-environment.py (keep them in sync). - var_assign_re = re.compile(r'(^|^declare\s+-\S+\s+|^declare\s+|^export\s+)([^=\s]+)=("|\')?(.*)$') - close_quote_re = re.compile(r'(\\"|"|\')\s*$') - def have_end_quote(quote, line): - close_quote_match = close_quote_re.search(line) - return close_quote_match is not None and \ - close_quote_match.group(1) == quote - - variables = frozenset(variables) - results = {} - for line in proc.stdout: - line = _unicode_decode(line, - encoding=_encodings['content'], errors='replace') - var_assign_match = var_assign_re.match(line) - if var_assign_match is not None: - key = var_assign_match.group(2) - quote = var_assign_match.group(3) - if quote is not None: - if have_end_quote(quote, - line[var_assign_match.end(2)+2:]): - value = var_assign_match.group(4) - else: - value = [var_assign_match.group(4)] - for line in proc.stdout: - line = _unicode_decode(line, - encoding=_encodings['content'], - errors='replace') - value.append(line) - if have_end_quote(quote, line): - break - value = ''.join(value) - # remove trailing quote and whitespace - value = value.rstrip()[:-1] - else: - value = var_assign_match.group(4).rstrip() - - if key in variables: - results[key] = value - - proc.wait() - proc.stdout.close() - return results - - def aux_update(self, cpv, values): - mylink = self._dblink(cpv) - if not mylink.exists(): - raise KeyError(cpv) - self._bump_mtime(cpv) - self._clear_pkg_cache(mylink) - for k, v in values.items(): - if v: - mylink.setfile(k, v) - else: - try: - os.unlink(os.path.join(self.getpath(cpv), k)) - except EnvironmentError: - pass - self._bump_mtime(cpv) - - def counter_tick(self, myroot=None, mycpv=None): - """ - @param myroot: ignored, self._eroot is used instead - """ - return self.counter_tick_core(incrementing=1, mycpv=mycpv) - - def get_counter_tick_core(self, myroot=None, mycpv=None): - """ - Use this method to retrieve the counter instead - of having to trust the value of a global counter - file that can lead to invalid COUNTER - generation. When cache is valid, the package COUNTER - files are not read and we rely on the timestamp of - the package directory to validate cache. The stat - calls should only take a short time, so performance - is sufficient without having to rely on a potentially - corrupt global counter file. - - The global counter file located at - $CACHE_PATH/counter serves to record the - counter of the last installed package and - it also corresponds to the total number of - installation actions that have occurred in - the history of this package database. - - @param myroot: ignored, self._eroot is used instead - """ - del myroot - counter = -1 - try: - with io.open( - _unicode_encode(self._counter_path, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], - errors='replace') as f: - try: - counter = long(f.readline().strip()) - except (OverflowError, ValueError) as e: - writemsg(_("!!! COUNTER file is corrupt: '%s'\n") % - self._counter_path, noiselevel=-1) - writemsg("!!! %s\n" % (e,), noiselevel=-1) - except EnvironmentError as e: - # Silently allow ENOENT since files under - # /var/cache/ are allowed to disappear. - if e.errno != errno.ENOENT: - writemsg(_("!!! Unable to read COUNTER file: '%s'\n") % \ - self._counter_path, noiselevel=-1) - writemsg("!!! %s\n" % str(e), noiselevel=-1) - del e - - if self._cached_counter == counter: - max_counter = counter - else: - # We must ensure that we return a counter - # value that is at least as large as the - # highest one from the installed packages, - # since having a corrupt value that is too low - # can trigger incorrect AUTOCLEAN behavior due - # to newly installed packages having lower - # COUNTERs than the previous version in the - # same slot. - max_counter = counter - for cpv in self.cpv_all(): - try: - pkg_counter = int(self.aux_get(cpv, ["COUNTER"])[0]) - except (KeyError, OverflowError, ValueError): - continue - if pkg_counter > max_counter: - max_counter = pkg_counter - - return max_counter + 1 - - def counter_tick_core(self, myroot=None, incrementing=1, mycpv=None): - """ - This method will grab the next COUNTER value and record it back - to the global file. Note that every package install must have - a unique counter, since a slotmove update can move two packages - into the same SLOT and in that case it's important that both - packages have different COUNTER metadata. - - @param myroot: ignored, self._eroot is used instead - @param mycpv: ignored - @rtype: int - @return: new counter value - """ - myroot = None - mycpv = None - self.lock() - try: - counter = self.get_counter_tick_core() - 1 - if incrementing: - #increment counter - counter += 1 - # update new global counter file - try: - write_atomic(self._counter_path, str(counter)) - except InvalidLocation: - self.settings._init_dirs() - write_atomic(self._counter_path, str(counter)) - self._cached_counter = counter - - # Since we hold a lock, this is a good opportunity - # to flush the cache. Note that this will only - # flush the cache periodically in the main process - # when _aux_cache_threshold is exceeded. - self.flush_cache() - finally: - self.unlock() - - return counter - - def _dblink(self, cpv): - category, pf = catsplit(cpv) - return dblink(category, pf, settings=self.settings, - vartree=self.vartree, treetype="vartree") - - def removeFromContents(self, pkg, paths, relative_paths=True): - """ - @param pkg: cpv for an installed package - @type pkg: string - @param paths: paths of files to remove from contents - @type paths: iterable - """ - if not hasattr(pkg, "getcontents"): - pkg = self._dblink(pkg) - root = self.settings['ROOT'] - root_len = len(root) - 1 - new_contents = pkg.getcontents().copy() - removed = 0 - - for filename in paths: - filename = _unicode_decode(filename, - encoding=_encodings['content'], errors='strict') - filename = normalize_path(filename) - if relative_paths: - relative_filename = filename - else: - relative_filename = filename[root_len:] - contents_key = pkg._match_contents(relative_filename) - if contents_key: - # It's possible for two different paths to refer to the same - # contents_key, due to directory symlinks. Therefore, pass a - # default value to pop, in order to avoid a KeyError which - # could otherwise be triggered (see bug #454400). - new_contents.pop(contents_key, None) - removed += 1 - - if removed: - # Also remove corresponding NEEDED lines, so that they do - # no corrupt LinkageMap data for preserve-libs. - needed_filename = os.path.join(pkg.dbdir, LinkageMap._needed_aux_key) - new_needed = None - try: - with io.open(_unicode_encode(needed_filename, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], - errors='replace') as f: - needed_lines = f.readlines() - except IOError as e: - if e.errno not in (errno.ENOENT, errno.ESTALE): - raise - else: - new_needed = [] - for l in needed_lines: - l = l.rstrip("\n") - if not l: - continue - try: - entry = NeededEntry.parse(needed_filename, l) - except InvalidData as e: - writemsg_level("\n%s\n\n" % (e,), - level=logging.ERROR, noiselevel=-1) - continue - - filename = os.path.join(root, entry.filename.lstrip(os.sep)) - if filename in new_contents: - new_needed.append(entry) - - self.writeContentsToContentsFile(pkg, new_contents, new_needed=new_needed) - - def writeContentsToContentsFile(self, pkg, new_contents, new_needed=None): - """ - @param pkg: package to write contents file for - @type pkg: dblink - @param new_contents: contents to write to CONTENTS file - @type new_contents: contents dictionary of the form - {u'/path/to/file' : (contents_attribute 1, ...), ...} - @param new_needed: new NEEDED entries - @type new_needed: list of NeededEntry - """ - root = self.settings['ROOT'] - self._bump_mtime(pkg.mycpv) - if new_needed is not None: - f = atomic_ofstream(os.path.join(pkg.dbdir, LinkageMap._needed_aux_key)) - for entry in new_needed: - f.write(_unicode(entry)) - f.close() - f = atomic_ofstream(os.path.join(pkg.dbdir, "CONTENTS")) - write_contents(new_contents, root, f) - f.close() - self._bump_mtime(pkg.mycpv) - pkg._clear_contents_cache() - - class _owners_cache(object): - """ - This class maintains an hash table that serves to index package - contents by mapping the basename of file to a list of possible - packages that own it. This is used to optimize owner lookups - by narrowing the search down to a smaller number of packages. - """ - _new_hash = md5 - _hash_bits = 16 - _hex_chars = _hash_bits // 4 - - def __init__(self, vardb): - self._vardb = vardb - - def add(self, cpv): - eroot_len = len(self._vardb._eroot) - pkg_hash = self._hash_pkg(cpv) - db = self._vardb._dblink(cpv) - if not db.getcontents(): - # Empty path is a code used to represent empty contents. - self._add_path("", pkg_hash) - - for x in db._contents.keys(): - self._add_path(x[eroot_len:], pkg_hash) - - self._vardb._aux_cache["modified"].add(cpv) - - def _add_path(self, path, pkg_hash): - """ - Empty path is a code that represents empty contents. - """ - if path: - name = os.path.basename(path.rstrip(os.path.sep)) - if not name: - return - else: - name = path - name_hash = self._hash_str(name) - base_names = self._vardb._aux_cache["owners"]["base_names"] - pkgs = base_names.get(name_hash) - if pkgs is None: - pkgs = {} - base_names[name_hash] = pkgs - pkgs[pkg_hash] = None - - def _hash_str(self, s): - h = self._new_hash() - # Always use a constant utf_8 encoding here, since - # the "default" encoding can change. - h.update(_unicode_encode(s, - encoding=_encodings['repo.content'], - errors='backslashreplace')) - h = h.hexdigest() - h = h[-self._hex_chars:] - h = int(h, 16) - return h - - def _hash_pkg(self, cpv): - counter, mtime = self._vardb.aux_get( - cpv, ["COUNTER", "_mtime_"]) - try: - counter = int(counter) - except ValueError: - counter = 0 - return (_unicode(cpv), counter, mtime) - - class _owners_db(object): - - def __init__(self, vardb): - self._vardb = vardb - - def populate(self): - self._populate() - - def _populate(self): - owners_cache = vardbapi._owners_cache(self._vardb) - cached_hashes = set() - base_names = self._vardb._aux_cache["owners"]["base_names"] - - # Take inventory of all cached package hashes. - for name, hash_values in list(base_names.items()): - if not isinstance(hash_values, dict): - del base_names[name] - continue - cached_hashes.update(hash_values) - - # Create sets of valid package hashes and uncached packages. - uncached_pkgs = set() - hash_pkg = owners_cache._hash_pkg - valid_pkg_hashes = set() - for cpv in self._vardb.cpv_all(): - hash_value = hash_pkg(cpv) - valid_pkg_hashes.add(hash_value) - if hash_value not in cached_hashes: - uncached_pkgs.add(cpv) - - # Cache any missing packages. - for cpv in uncached_pkgs: - owners_cache.add(cpv) - - # Delete any stale cache. - stale_hashes = cached_hashes.difference(valid_pkg_hashes) - if stale_hashes: - for base_name_hash, bucket in list(base_names.items()): - for hash_value in stale_hashes.intersection(bucket): - del bucket[hash_value] - if not bucket: - del base_names[base_name_hash] - - return owners_cache - - def get_owners(self, path_iter): - """ - @return the owners as a dblink -> set(files) mapping. - """ - owners = {} - for owner, f in self.iter_owners(path_iter): - owned_files = owners.get(owner) - if owned_files is None: - owned_files = set() - owners[owner] = owned_files - owned_files.add(f) - return owners - - def getFileOwnerMap(self, path_iter): - owners = self.get_owners(path_iter) - file_owners = {} - for pkg_dblink, files in owners.items(): - for f in files: - owner_set = file_owners.get(f) - if owner_set is None: - owner_set = set() - file_owners[f] = owner_set - owner_set.add(pkg_dblink) - return file_owners - - def iter_owners(self, path_iter): - """ - Iterate over tuples of (dblink, path). In order to avoid - consuming too many resources for too much time, resources - are only allocated for the duration of a given iter_owners() - call. Therefore, to maximize reuse of resources when searching - for multiple files, it's best to search for them all in a single - call. - """ - - if not isinstance(path_iter, list): - path_iter = list(path_iter) - owners_cache = self._populate() - vardb = self._vardb - root = vardb._eroot - hash_pkg = owners_cache._hash_pkg - hash_str = owners_cache._hash_str - base_names = self._vardb._aux_cache["owners"]["base_names"] - case_insensitive = "case-insensitive-fs" \ - in vardb.settings.features - - dblink_cache = {} - - def dblink(cpv): - x = dblink_cache.get(cpv) - if x is None: - if len(dblink_cache) > 20: - # Ensure that we don't run out of memory. - raise StopIteration() - x = self._vardb._dblink(cpv) - dblink_cache[cpv] = x - return x - - while path_iter: - - path = path_iter.pop() - if case_insensitive: - path = path.lower() - is_basename = os.sep != path[:1] - if is_basename: - name = path - else: - name = os.path.basename(path.rstrip(os.path.sep)) - - if not name: - continue - - name_hash = hash_str(name) - pkgs = base_names.get(name_hash) - owners = [] - if pkgs is not None: - try: - for hash_value in pkgs: - if not isinstance(hash_value, tuple) or \ - len(hash_value) != 3: - continue - cpv, counter, mtime = hash_value - if not isinstance(cpv, basestring): - continue - try: - current_hash = hash_pkg(cpv) - except KeyError: - continue - - if current_hash != hash_value: - continue - - if is_basename: - for p in dblink(cpv)._contents.keys(): - if os.path.basename(p) == name: - owners.append((cpv, dblink(cpv). - _contents.unmap_key( - p)[len(root):])) - else: - key = dblink(cpv)._match_contents(path) - if key is not False: - owners.append( - (cpv, key[len(root):])) - - except StopIteration: - path_iter.append(path) - del owners[:] - dblink_cache.clear() - gc.collect() - for x in self._iter_owners_low_mem(path_iter): - yield x - return - else: - for cpv, p in owners: - yield (dblink(cpv), p) - - def _iter_owners_low_mem(self, path_list): - """ - This implemention will make a short-lived dblink instance (and - parse CONTENTS) for every single installed package. This is - slower and but uses less memory than the method which uses the - basename cache. - """ - - if not path_list: - return - - case_insensitive = "case-insensitive-fs" \ - in self._vardb.settings.features - path_info_list = [] - for path in path_list: - if case_insensitive: - path = path.lower() - is_basename = os.sep != path[:1] - if is_basename: - name = path - else: - name = os.path.basename(path.rstrip(os.path.sep)) - path_info_list.append((path, name, is_basename)) - - # Do work via the global event loop, so that it can be used - # for indication of progress during the search (bug #461412). - event_loop = (portage._internal_caller and - global_event_loop() or EventLoop(main=False)) - root = self._vardb._eroot - - def search_pkg(cpv, search_future): - dblnk = self._vardb._dblink(cpv) - results = [] - for path, name, is_basename in path_info_list: - if is_basename: - for p in dblnk._contents.keys(): - if os.path.basename(p) == name: - results.append((dblnk, - dblnk._contents.unmap_key( - p)[len(root):])) - else: - key = dblnk._match_contents(path) - if key is not False: - results.append( - (dblnk, key[len(root):])) - search_future.set_result(results) - - for cpv in self._vardb.cpv_all(): - search_future = event_loop.create_future() - event_loop.call_soon(search_pkg, cpv, search_future) - event_loop.run_until_complete(search_future) - for result in search_future.result(): - yield result - -class vartree(object): - "this tree will scan a var/db/pkg database located at root (passed to init)" - def __init__(self, root=None, virtual=DeprecationWarning, categories=None, - settings=None): - - if settings is None: - settings = portage.settings - - if root is not None and root != settings['ROOT']: - warnings.warn("The 'root' parameter of the " - "portage.dbapi.vartree.vartree" - " constructor is now unused. Use " - "settings['ROOT'] instead.", - DeprecationWarning, stacklevel=2) - - if virtual is not DeprecationWarning: - warnings.warn("The 'virtual' parameter of the " - "portage.dbapi.vartree.vartree" - " constructor is unused", - DeprecationWarning, stacklevel=2) - - self.settings = settings - self.dbapi = vardbapi(settings=settings, vartree=self) - self.populated = 1 - - @property - def root(self): - warnings.warn("The root attribute of " - "portage.dbapi.vartree.vartree" - " is deprecated. Use " - "settings['ROOT'] instead.", - DeprecationWarning, stacklevel=3) - return self.settings['ROOT'] - - def getpath(self, mykey, filename=None): - return self.dbapi.getpath(mykey, filename=filename) - - def zap(self, mycpv): - return - - def inject(self, mycpv): - return - - def get_provide(self, mycpv): - return [] - - def get_all_provides(self): - return {} - - def dep_bestmatch(self, mydep, use_cache=1): - "compatibility method -- all matches, not just visible ones" - #mymatch=best(match(dep_expand(mydep,self.dbapi),self.dbapi)) - mymatch = best(self.dbapi.match( - dep_expand(mydep, mydb=self.dbapi, settings=self.settings), - use_cache=use_cache)) - if mymatch is None: - return "" - else: - return mymatch - - def dep_match(self, mydep, use_cache=1): - "compatibility method -- we want to see all matches, not just visible ones" - #mymatch = match(mydep,self.dbapi) - mymatch = self.dbapi.match(mydep, use_cache=use_cache) - if mymatch is None: - return [] - else: - return mymatch - - def exists_specific(self, cpv): - return self.dbapi.cpv_exists(cpv) - - def getallcpv(self): - """temporary function, probably to be renamed --- Gets a list of all - category/package-versions installed on the system.""" - return self.dbapi.cpv_all() - - def getallnodes(self): - """new behavior: these are all *unmasked* nodes. There may or may not be available - masked package for nodes in this nodes list.""" - return self.dbapi.cp_all() - - def getebuildpath(self, fullpackage): - cat, package = catsplit(fullpackage) - return self.getpath(fullpackage, filename=package+".ebuild") - - def getslot(self, mycatpkg): - "Get a slot for a catpkg; assume it exists." - try: - return self.dbapi._pkg_str(mycatpkg, None).slot - except KeyError: - return "" - - def populate(self): - self.populated=1 - -class dblink(object): - """ - This class provides an interface to the installed package database - At present this is implemented as a text backend in /var/db/pkg. - """ - - import re - _normalize_needed = re.compile(r'//|^[^/]|./$|(^|/)\.\.?(/|$)') - - _contents_re = re.compile(r'^(' + \ - r'(?P(dev|dir|fif) (.+))|' + \ - r'(?P(obj) (.+) (\S+) (\d+))|' + \ - r'(?P(sym) (.+) -> (.+) ((\d+)|(?P(' + \ - r'\(\d+, \d+L, \d+L, \d+, \d+, \d+, \d+L, \d+, (\d+), \d+\)))))' + \ - r')$' - ) - - # These files are generated by emerge, so we need to remove - # them when they are the only thing left in a directory. - _infodir_cleanup = frozenset(["dir", "dir.old"]) - - _ignored_unlink_errnos = ( - errno.EBUSY, errno.ENOENT, - errno.ENOTDIR, errno.EISDIR) - - _ignored_rmdir_errnos = ( - errno.EEXIST, errno.ENOTEMPTY, - errno.EBUSY, errno.ENOENT, - errno.ENOTDIR, errno.EISDIR, - errno.EPERM) - - def __init__(self, cat, pkg, myroot=None, settings=None, treetype=None, - vartree=None, blockers=None, scheduler=None, pipe=None): - """ - Creates a DBlink object for a given CPV. - The given CPV may not be present in the database already. - - @param cat: Category - @type cat: String - @param pkg: Package (PV) - @type pkg: String - @param myroot: ignored, settings['ROOT'] is used instead - @type myroot: String (Path) - @param settings: Typically portage.settings - @type settings: portage.config - @param treetype: one of ['porttree','bintree','vartree'] - @type treetype: String - @param vartree: an instance of vartree corresponding to myroot. - @type vartree: vartree - """ - - if settings is None: - raise TypeError("settings argument is required") - - mysettings = settings - self._eroot = mysettings['EROOT'] - self.cat = cat - self.pkg = pkg - self.mycpv = self.cat + "/" + self.pkg - if self.mycpv == settings.mycpv and \ - isinstance(settings.mycpv, _pkg_str): - self.mycpv = settings.mycpv - else: - self.mycpv = _pkg_str(self.mycpv) - self.mysplit = list(self.mycpv.cpv_split[1:]) - self.mysplit[0] = self.mycpv.cp - self.treetype = treetype - if vartree is None: - vartree = portage.db[self._eroot]["vartree"] - self.vartree = vartree - self._blockers = blockers - self._scheduler = scheduler - self.dbroot = normalize_path(os.path.join(self._eroot, VDB_PATH)) - self.dbcatdir = self.dbroot+"/"+cat - self.dbpkgdir = self.dbcatdir+"/"+pkg - self.dbtmpdir = self.dbcatdir+"/"+MERGING_IDENTIFIER+pkg - self.dbdir = self.dbpkgdir - self.settings = mysettings - self._verbose = self.settings.get("PORTAGE_VERBOSE") == "1" - - self.myroot = self.settings['ROOT'] - self._installed_instance = None - self.contentscache = None - self._contents_inodes = None - self._contents_basenames = None - self._linkmap_broken = False - self._device_path_map = {} - self._hardlink_merge_map = {} - self._hash_key = (self._eroot, self.mycpv) - self._protect_obj = None - self._pipe = pipe - self._postinst_failure = False - - # When necessary, this attribute is modified for - # compliance with RESTRICT=preserve-libs. - self._preserve_libs = "preserve-libs" in mysettings.features - self._contents = ContentsCaseSensitivityManager(self) - self._slot_locks = [] - - def __hash__(self): - return hash(self._hash_key) - - def __eq__(self, other): - return isinstance(other, dblink) and \ - self._hash_key == other._hash_key - - def _get_protect_obj(self): - - if self._protect_obj is None: - self._protect_obj = ConfigProtect(self._eroot, - portage.util.shlex_split( - self.settings.get("CONFIG_PROTECT", "")), - portage.util.shlex_split( - self.settings.get("CONFIG_PROTECT_MASK", "")), - case_insensitive=("case-insensitive-fs" - in self.settings.features)) - - return self._protect_obj - - def isprotected(self, obj): - return self._get_protect_obj().isprotected(obj) - - def updateprotect(self): - self._get_protect_obj().updateprotect() - - def lockdb(self): - self.vartree.dbapi.lock() - - def unlockdb(self): - self.vartree.dbapi.unlock() - - def _slot_locked(f): - """ - A decorator function which, when parallel-install is enabled, - acquires and releases slot locks for the current package and - blocked packages. This is required in order to account for - interactions with blocked packages (involving resolution of - file collisions). - """ - def wrapper(self, *args, **kwargs): - if "parallel-install" in self.settings.features: - self._acquire_slot_locks( - kwargs.get("mydbapi", self.vartree.dbapi)) - try: - return f(self, *args, **kwargs) - finally: - self._release_slot_locks() - return wrapper - - def _acquire_slot_locks(self, db): - """ - Acquire slot locks for the current package and blocked packages. - """ - - slot_atoms = [] - - try: - slot = self.mycpv.slot - except AttributeError: - slot, = db.aux_get(self.mycpv, ["SLOT"]) - slot = slot.partition("/")[0] - - slot_atoms.append(portage.dep.Atom( - "%s:%s" % (self.mycpv.cp, slot))) - - for blocker in self._blockers or []: - slot_atoms.append(blocker.slot_atom) - - # Sort atoms so that locks are acquired in a predictable - # order, preventing deadlocks with competitors that may - # be trying to acquire overlapping locks. - slot_atoms.sort() - for slot_atom in slot_atoms: - self.vartree.dbapi._slot_lock(slot_atom) - self._slot_locks.append(slot_atom) - - def _release_slot_locks(self): - """ - Release all slot locks. - """ - while self._slot_locks: - self.vartree.dbapi._slot_unlock(self._slot_locks.pop()) - - def getpath(self): - "return path to location of db information (for >>> informational display)" - return self.dbdir - - def exists(self): - "does the db entry exist? boolean." - return os.path.exists(self.dbdir) - - def delete(self): - """ - Remove this entry from the database - """ - try: - os.lstat(self.dbdir) - except OSError as e: - if e.errno not in (errno.ENOENT, errno.ENOTDIR, errno.ESTALE): - raise - return - - # Check validity of self.dbdir before attempting to remove it. - if not self.dbdir.startswith(self.dbroot): - writemsg(_("portage.dblink.delete(): invalid dbdir: %s\n") % \ - self.dbdir, noiselevel=-1) - return - - if self.dbdir is self.dbpkgdir: - counter, = self.vartree.dbapi.aux_get( - self.mycpv, ["COUNTER"]) - self.vartree.dbapi._cache_delta.recordEvent( - "remove", self.mycpv, - self.settings["SLOT"].split("/")[0], counter) - - shutil.rmtree(self.dbdir) - # If empty, remove parent category directory. - try: - os.rmdir(os.path.dirname(self.dbdir)) - except OSError: - pass - self.vartree.dbapi._remove(self) - - # Use self.dbroot since we need an existing path for syncfs. - try: - self._merged_path(self.dbroot, os.lstat(self.dbroot)) - except OSError: - pass - - self._post_merge_sync() - - def clearcontents(self): - """ - For a given db entry (self), erase the CONTENTS values. - """ - self.lockdb() - try: - if os.path.exists(self.dbdir+"/CONTENTS"): - os.unlink(self.dbdir+"/CONTENTS") - finally: - self.unlockdb() - - def _clear_contents_cache(self): - self.contentscache = None - self._contents_inodes = None - self._contents_basenames = None - self._contents.clear_cache() - - def getcontents(self): - """ - Get the installed files of a given package (aka what that package installed) - """ - if self.contentscache is not None: - return self.contentscache - contents_file = os.path.join(self.dbdir, "CONTENTS") - pkgfiles = {} - try: - with io.open(_unicode_encode(contents_file, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], - errors='replace') as f: - mylines = f.readlines() - except EnvironmentError as e: - if e.errno != errno.ENOENT: - raise - del e - self.contentscache = pkgfiles - return pkgfiles - - null_byte = "\0" - normalize_needed = self._normalize_needed - contents_re = self._contents_re - obj_index = contents_re.groupindex['obj'] - dir_index = contents_re.groupindex['dir'] - sym_index = contents_re.groupindex['sym'] - # The old symlink format may exist on systems that have packages - # which were installed many years ago (see bug #351814). - oldsym_index = contents_re.groupindex['oldsym'] - # CONTENTS files already contain EPREFIX - myroot = self.settings['ROOT'] - if myroot == os.path.sep: - myroot = None - # used to generate parent dir entries - dir_entry = ("dir",) - eroot_split_len = len(self.settings["EROOT"].split(os.sep)) - 1 - pos = 0 - errors = [] - for pos, line in enumerate(mylines): - if null_byte in line: - # Null bytes are a common indication of corruption. - errors.append((pos + 1, _("Null byte found in CONTENTS entry"))) - continue - line = line.rstrip("\n") - m = contents_re.match(line) - if m is None: - errors.append((pos + 1, _("Unrecognized CONTENTS entry"))) - continue - - if m.group(obj_index) is not None: - base = obj_index - #format: type, mtime, md5sum - data = (m.group(base+1), m.group(base+4), m.group(base+3)) - elif m.group(dir_index) is not None: - base = dir_index - #format: type - data = (m.group(base+1),) - elif m.group(sym_index) is not None: - base = sym_index - if m.group(oldsym_index) is None: - mtime = m.group(base+5) - else: - mtime = m.group(base+8) - #format: type, mtime, dest - data = (m.group(base+1), mtime, m.group(base+3)) - else: - # This won't happen as long the regular expression - # is written to only match valid entries. - raise AssertionError(_("required group not found " + \ - "in CONTENTS entry: '%s'") % line) - - path = m.group(base+2) - if normalize_needed.search(path) is not None: - path = normalize_path(path) - if not path.startswith(os.path.sep): - path = os.path.sep + path - - if myroot is not None: - path = os.path.join(myroot, path.lstrip(os.path.sep)) - - # Implicitly add parent directories, since we can't necessarily - # assume that they are explicitly listed in CONTENTS, and it's - # useful for callers if they can rely on parent directory entries - # being generated here (crucial for things like dblink.isowner()). - path_split = path.split(os.sep) - path_split.pop() - while len(path_split) > eroot_split_len: - parent = os.sep.join(path_split) - if parent in pkgfiles: - break - pkgfiles[parent] = dir_entry - path_split.pop() - - pkgfiles[path] = data - - if errors: - writemsg(_("!!! Parse error in '%s'\n") % contents_file, noiselevel=-1) - for pos, e in errors: - writemsg(_("!!! line %d: %s\n") % (pos, e), noiselevel=-1) - self.contentscache = pkgfiles - return pkgfiles - - def _prune_plib_registry(self, unmerge=False, - needed=None, preserve_paths=None): - # remove preserved libraries that don't have any consumers left - if not (self._linkmap_broken or - self.vartree.dbapi._linkmap is None or - self.vartree.dbapi._plib_registry is None): - self.vartree.dbapi._fs_lock() - plib_registry = self.vartree.dbapi._plib_registry - plib_registry.lock() - try: - plib_registry.load() - - unmerge_with_replacement = \ - unmerge and preserve_paths is not None - if unmerge_with_replacement: - # If self.mycpv is about to be unmerged and we - # have a replacement package, we want to exclude - # the irrelevant NEEDED data that belongs to - # files which are being unmerged now. - exclude_pkgs = (self.mycpv,) - else: - exclude_pkgs = None - - self._linkmap_rebuild(exclude_pkgs=exclude_pkgs, - include_file=needed, preserve_paths=preserve_paths) - - if unmerge: - unmerge_preserve = None - if not unmerge_with_replacement: - unmerge_preserve = \ - self._find_libs_to_preserve(unmerge=True) - counter = self.vartree.dbapi.cpv_counter(self.mycpv) - try: - slot = self.mycpv.slot - except AttributeError: - slot = _pkg_str(self.mycpv, slot=self.settings["SLOT"]).slot - plib_registry.unregister(self.mycpv, slot, counter) - if unmerge_preserve: - for path in sorted(unmerge_preserve): - contents_key = self._match_contents(path) - if not contents_key: - continue - obj_type = self.getcontents()[contents_key][0] - self._display_merge(_(">>> needed %s %s\n") % \ - (obj_type, contents_key), noiselevel=-1) - plib_registry.register(self.mycpv, - slot, counter, unmerge_preserve) - # Remove the preserved files from our contents - # so that they won't be unmerged. - self.vartree.dbapi.removeFromContents(self, - unmerge_preserve) - - unmerge_no_replacement = \ - unmerge and not unmerge_with_replacement - cpv_lib_map = self._find_unused_preserved_libs( - unmerge_no_replacement) - if cpv_lib_map: - self._remove_preserved_libs(cpv_lib_map) - self.vartree.dbapi.lock() - try: - for cpv, removed in cpv_lib_map.items(): - if not self.vartree.dbapi.cpv_exists(cpv): - continue - self.vartree.dbapi.removeFromContents(cpv, removed) - finally: - self.vartree.dbapi.unlock() - - plib_registry.store() - finally: - plib_registry.unlock() - self.vartree.dbapi._fs_unlock() - - @_slot_locked - def unmerge(self, pkgfiles=None, trimworld=None, cleanup=True, - ldpath_mtimes=None, others_in_slot=None, needed=None, - preserve_paths=None): - """ - Calls prerm - Unmerges a given package (CPV) - calls postrm - calls cleanrm - calls env_update - - @param pkgfiles: files to unmerge (generally self.getcontents() ) - @type pkgfiles: Dictionary - @param trimworld: Unused - @type trimworld: Boolean - @param cleanup: cleanup to pass to doebuild (see doebuild) - @type cleanup: Boolean - @param ldpath_mtimes: mtimes to pass to env_update (see env_update) - @type ldpath_mtimes: Dictionary - @param others_in_slot: all dblink instances in this slot, excluding self - @type others_in_slot: list - @param needed: Filename containing libraries needed after unmerge. - @type needed: String - @param preserve_paths: Libraries preserved by a package instance that - is currently being merged. They need to be explicitly passed to the - LinkageMap, since they are not registered in the - PreservedLibsRegistry yet. - @type preserve_paths: set - @rtype: Integer - @return: - 1. os.EX_OK if everything went well. - 2. return code of the failed phase (for prerm, postrm, cleanrm) - """ - - if trimworld is not None: - warnings.warn("The trimworld parameter of the " + \ - "portage.dbapi.vartree.dblink.unmerge()" + \ - " method is now unused.", - DeprecationWarning, stacklevel=2) - - background = False - log_path = self.settings.get("PORTAGE_LOG_FILE") - if self._scheduler is None: - # We create a scheduler instance and use it to - # log unmerge output separately from merge output. - self._scheduler = SchedulerInterface(portage._internal_caller and - global_event_loop() or EventLoop(main=False)) - if self.settings.get("PORTAGE_BACKGROUND") == "subprocess": - if self.settings.get("PORTAGE_BACKGROUND_UNMERGE") == "1": - self.settings["PORTAGE_BACKGROUND"] = "1" - self.settings.backup_changes("PORTAGE_BACKGROUND") - background = True - elif self.settings.get("PORTAGE_BACKGROUND_UNMERGE") == "0": - self.settings["PORTAGE_BACKGROUND"] = "0" - self.settings.backup_changes("PORTAGE_BACKGROUND") - elif self.settings.get("PORTAGE_BACKGROUND") == "1": - background = True - - self.vartree.dbapi._bump_mtime(self.mycpv) - showMessage = self._display_merge - if self.vartree.dbapi._categories is not None: - self.vartree.dbapi._categories = None - - # When others_in_slot is not None, the backup has already been - # handled by the caller. - caller_handles_backup = others_in_slot is not None - - # When others_in_slot is supplied, the security check has already been - # done for this slot, so it shouldn't be repeated until the next - # replacement or unmerge operation. - if others_in_slot is None: - slot = self.vartree.dbapi._pkg_str(self.mycpv, None).slot - slot_matches = self.vartree.dbapi.match( - "%s:%s" % (portage.cpv_getkey(self.mycpv), slot)) - others_in_slot = [] - for cur_cpv in slot_matches: - if cur_cpv == self.mycpv: - continue - others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1], - settings=self.settings, vartree=self.vartree, - treetype="vartree", pipe=self._pipe)) - - retval = self._security_check([self] + others_in_slot) - if retval: - return retval - - contents = self.getcontents() - # Now, don't assume that the name of the ebuild is the same as the - # name of the dir; the package may have been moved. - myebuildpath = os.path.join(self.dbdir, self.pkg + ".ebuild") - failures = 0 - ebuild_phase = "prerm" - mystuff = os.listdir(self.dbdir) - for x in mystuff: - if x.endswith(".ebuild"): - if x[:-7] != self.pkg: - # Clean up after vardbapi.move_ent() breakage in - # portage versions before 2.1.2 - os.rename(os.path.join(self.dbdir, x), myebuildpath) - write_atomic(os.path.join(self.dbdir, "PF"), self.pkg+"\n") - break - - if self.mycpv != self.settings.mycpv or \ - "EAPI" not in self.settings.configdict["pkg"]: - # We avoid a redundant setcpv call here when - # the caller has already taken care of it. - self.settings.setcpv(self.mycpv, mydb=self.vartree.dbapi) - - eapi_unsupported = False - try: - doebuild_environment(myebuildpath, "prerm", - settings=self.settings, db=self.vartree.dbapi) - except UnsupportedAPIException as e: - eapi_unsupported = e - - if self._preserve_libs and "preserve-libs" in \ - self.settings["PORTAGE_RESTRICT"].split(): - self._preserve_libs = False - - builddir_lock = None - scheduler = self._scheduler - retval = os.EX_OK - try: - # Only create builddir_lock if the caller - # has not already acquired the lock. - if "PORTAGE_BUILDDIR_LOCKED" not in self.settings: - builddir_lock = EbuildBuildDir( - scheduler=scheduler, - settings=self.settings) - scheduler.run_until_complete(builddir_lock.async_lock()) - prepare_build_dirs(settings=self.settings, cleanup=True) - log_path = self.settings.get("PORTAGE_LOG_FILE") - - # Do this before the following _prune_plib_registry call, since - # that removes preserved libraries from our CONTENTS, and we - # may want to backup those libraries first. - if not caller_handles_backup: - retval = self._pre_unmerge_backup(background) - if retval != os.EX_OK: - showMessage(_("!!! FAILED prerm: quickpkg: %s\n") % retval, - level=logging.ERROR, noiselevel=-1) - return retval - - self._prune_plib_registry(unmerge=True, needed=needed, - preserve_paths=preserve_paths) - - # Log the error after PORTAGE_LOG_FILE is initialized - # by prepare_build_dirs above. - if eapi_unsupported: - # Sometimes this happens due to corruption of the EAPI file. - failures += 1 - showMessage(_("!!! FAILED prerm: %s\n") % \ - os.path.join(self.dbdir, "EAPI"), - level=logging.ERROR, noiselevel=-1) - showMessage("%s\n" % (eapi_unsupported,), - level=logging.ERROR, noiselevel=-1) - elif os.path.isfile(myebuildpath): - phase = EbuildPhase(background=background, - phase=ebuild_phase, scheduler=scheduler, - settings=self.settings) - phase.start() - retval = phase.wait() - - # XXX: Decide how to handle failures here. - if retval != os.EX_OK: - failures += 1 - showMessage(_("!!! FAILED prerm: %s\n") % retval, - level=logging.ERROR, noiselevel=-1) - - self.vartree.dbapi._fs_lock() - try: - self._unmerge_pkgfiles(pkgfiles, others_in_slot) - finally: - self.vartree.dbapi._fs_unlock() - self._clear_contents_cache() - - if not eapi_unsupported and os.path.isfile(myebuildpath): - ebuild_phase = "postrm" - phase = EbuildPhase(background=background, - phase=ebuild_phase, scheduler=scheduler, - settings=self.settings) - phase.start() - retval = phase.wait() - - # XXX: Decide how to handle failures here. - if retval != os.EX_OK: - failures += 1 - showMessage(_("!!! FAILED postrm: %s\n") % retval, - level=logging.ERROR, noiselevel=-1) - - finally: - self.vartree.dbapi._bump_mtime(self.mycpv) - try: - if not eapi_unsupported and os.path.isfile(myebuildpath): - if retval != os.EX_OK: - msg_lines = [] - msg = _("The '%(ebuild_phase)s' " - "phase of the '%(cpv)s' package " - "has failed with exit value %(retval)s.") % \ - {"ebuild_phase":ebuild_phase, "cpv":self.mycpv, - "retval":retval} - from textwrap import wrap - msg_lines.extend(wrap(msg, 72)) - msg_lines.append("") - - ebuild_name = os.path.basename(myebuildpath) - ebuild_dir = os.path.dirname(myebuildpath) - msg = _("The problem occurred while executing " - "the ebuild file named '%(ebuild_name)s' " - "located in the '%(ebuild_dir)s' directory. " - "If necessary, manually remove " - "the environment.bz2 file and/or the " - "ebuild file located in that directory.") % \ - {"ebuild_name":ebuild_name, "ebuild_dir":ebuild_dir} - msg_lines.extend(wrap(msg, 72)) - msg_lines.append("") - - msg = _("Removal " - "of the environment.bz2 file is " - "preferred since it may allow the " - "removal phases to execute successfully. " - "The ebuild will be " - "sourced and the eclasses " - "from the current portage tree will be used " - "when necessary. Removal of " - "the ebuild file will cause the " - "pkg_prerm() and pkg_postrm() removal " - "phases to be skipped entirely.") - msg_lines.extend(wrap(msg, 72)) - - self._eerror(ebuild_phase, msg_lines) - - self._elog_process(phasefilter=("prerm", "postrm")) - - if retval == os.EX_OK: - try: - doebuild_environment(myebuildpath, "cleanrm", - settings=self.settings, db=self.vartree.dbapi) - except UnsupportedAPIException: - pass - phase = EbuildPhase(background=background, - phase="cleanrm", scheduler=scheduler, - settings=self.settings) - phase.start() - retval = phase.wait() - finally: - if builddir_lock is not None: - scheduler.run_until_complete( - builddir_lock.async_unlock()) - - if log_path is not None: - - if not failures and 'unmerge-logs' not in self.settings.features: - try: - os.unlink(log_path) - except OSError: - pass - - try: - st = os.stat(log_path) - except OSError: - pass - else: - if st.st_size == 0: - try: - os.unlink(log_path) - except OSError: - pass - - if log_path is not None and os.path.exists(log_path): - # Restore this since it gets lost somewhere above and it - # needs to be set for _display_merge() to be able to log. - # Note that the log isn't necessarily supposed to exist - # since if PORT_LOGDIR is unset then it's a temp file - # so it gets cleaned above. - self.settings["PORTAGE_LOG_FILE"] = log_path - else: - self.settings.pop("PORTAGE_LOG_FILE", None) - - env_update(target_root=self.settings['ROOT'], - prev_mtimes=ldpath_mtimes, - contents=contents, env=self.settings, - writemsg_level=self._display_merge, vardbapi=self.vartree.dbapi) - - unmerge_with_replacement = preserve_paths is not None - if not unmerge_with_replacement: - # When there's a replacement package which calls us via treewalk, - # treewalk will automatically call _prune_plib_registry for us. - # Otherwise, we need to call _prune_plib_registry ourselves. - # Don't pass in the "unmerge=True" flag here, since that flag - # is intended to be used _prior_ to unmerge, not after. - self._prune_plib_registry() - - return os.EX_OK - - def _display_merge(self, msg, level=0, noiselevel=0): - if not self._verbose and noiselevel >= 0 and level < logging.WARN: - return - if self._scheduler is None: - writemsg_level(msg, level=level, noiselevel=noiselevel) - else: - log_path = None - if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": - log_path = self.settings.get("PORTAGE_LOG_FILE") - background = self.settings.get("PORTAGE_BACKGROUND") == "1" - - if background and log_path is None: - if level >= logging.WARN: - writemsg_level(msg, level=level, noiselevel=noiselevel) - else: - self._scheduler.output(msg, - log_path=log_path, background=background, - level=level, noiselevel=noiselevel) - - def _show_unmerge(self, zing, desc, file_type, file_name): - self._display_merge("%s %s %s %s\n" % \ - (zing, desc.ljust(8), file_type, file_name)) - - def _unmerge_pkgfiles(self, pkgfiles, others_in_slot): - """ - - Unmerges the contents of a package from the liveFS - Removes the VDB entry for self - - @param pkgfiles: typically self.getcontents() - @type pkgfiles: Dictionary { filename: [ 'type', '?', 'md5sum' ] } - @param others_in_slot: all dblink instances in this slot, excluding self - @type others_in_slot: list - @rtype: None - """ - - os = _os_merge - perf_md5 = perform_md5 - showMessage = self._display_merge - show_unmerge = self._show_unmerge - ignored_unlink_errnos = self._ignored_unlink_errnos - ignored_rmdir_errnos = self._ignored_rmdir_errnos - - if not pkgfiles: - showMessage(_("No package files given... Grabbing a set.\n")) - pkgfiles = self.getcontents() - - if others_in_slot is None: - others_in_slot = [] - slot = self.vartree.dbapi._pkg_str(self.mycpv, None).slot - slot_matches = self.vartree.dbapi.match( - "%s:%s" % (portage.cpv_getkey(self.mycpv), slot)) - for cur_cpv in slot_matches: - if cur_cpv == self.mycpv: - continue - others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1], - settings=self.settings, - vartree=self.vartree, treetype="vartree", pipe=self._pipe)) - - cfgfiledict = grabdict(self.vartree.dbapi._conf_mem_file) - stale_confmem = [] - protected_symlinks = {} - - unmerge_orphans = "unmerge-orphans" in self.settings.features - calc_prelink = "prelink-checksums" in self.settings.features - - if pkgfiles: - self.updateprotect() - mykeys = list(pkgfiles) - mykeys.sort() - mykeys.reverse() - - #process symlinks second-to-last, directories last. - mydirs = set() - - uninstall_ignore = portage.util.shlex_split( - self.settings.get("UNINSTALL_IGNORE", "")) - - def unlink(file_name, lstatobj): - if bsd_chflags: - if lstatobj.st_flags != 0: - bsd_chflags.lchflags(file_name, 0) - parent_name = os.path.dirname(file_name) - # Use normal stat/chflags for the parent since we want to - # follow any symlinks to the real parent directory. - pflags = os.stat(parent_name).st_flags - if pflags != 0: - bsd_chflags.chflags(parent_name, 0) - try: - if not stat.S_ISLNK(lstatobj.st_mode): - # Remove permissions to ensure that any hardlinks to - # suid/sgid files are rendered harmless. - os.chmod(file_name, 0) - os.unlink(file_name) - except OSError as ose: - # If the chmod or unlink fails, you are in trouble. - # With Prefix this can be because the file is owned - # by someone else (a screwup by root?), on a normal - # system maybe filesystem corruption. In any case, - # if we backtrace and die here, we leave the system - # in a totally undefined state, hence we just bleed - # like hell and continue to hopefully finish all our - # administrative and pkg_postinst stuff. - self._eerror("postrm", - ["Could not chmod or unlink '%s': %s" % \ - (file_name, ose)]) - else: - - # Even though the file no longer exists, we log it - # here so that _unmerge_dirs can see that we've - # removed a file from this device, and will record - # the parent directory for a syncfs call. - self._merged_path(file_name, lstatobj, exists=False) - - finally: - if bsd_chflags and pflags != 0: - # Restore the parent flags we saved before unlinking - bsd_chflags.chflags(parent_name, pflags) - - unmerge_desc = {} - unmerge_desc["cfgpro"] = _("cfgpro") - unmerge_desc["replaced"] = _("replaced") - unmerge_desc["!dir"] = _("!dir") - unmerge_desc["!empty"] = _("!empty") - unmerge_desc["!fif"] = _("!fif") - unmerge_desc["!found"] = _("!found") - unmerge_desc["!md5"] = _("!md5") - unmerge_desc["!mtime"] = _("!mtime") - unmerge_desc["!obj"] = _("!obj") - unmerge_desc["!sym"] = _("!sym") - unmerge_desc["!prefix"] = _("!prefix") - - real_root = self.settings['ROOT'] - real_root_len = len(real_root) - 1 - eroot = self.settings["EROOT"] - - infodirs = frozenset(infodir for infodir in chain( - self.settings.get("INFOPATH", "").split(":"), - self.settings.get("INFODIR", "").split(":")) if infodir) - infodirs_inodes = set() - for infodir in infodirs: - infodir = os.path.join(real_root, infodir.lstrip(os.sep)) - try: - statobj = os.stat(infodir) - except OSError: - pass - else: - infodirs_inodes.add((statobj.st_dev, statobj.st_ino)) - - for i, objkey in enumerate(mykeys): - - obj = normalize_path(objkey) - if os is _os_merge: - try: - _unicode_encode(obj, - encoding=_encodings['merge'], errors='strict') - except UnicodeEncodeError: - # The package appears to have been merged with a - # different value of sys.getfilesystemencoding(), - # so fall back to utf_8 if appropriate. - try: - _unicode_encode(obj, - encoding=_encodings['fs'], errors='strict') - except UnicodeEncodeError: - pass - else: - os = portage.os - perf_md5 = portage.checksum.perform_md5 - - file_data = pkgfiles[objkey] - file_type = file_data[0] - - # don't try to unmerge the prefix offset itself - if len(obj) <= len(eroot) or not obj.startswith(eroot): - show_unmerge("---", unmerge_desc["!prefix"], file_type, obj) - continue - - statobj = None - try: - statobj = os.stat(obj) - except OSError: - pass - lstatobj = None - try: - lstatobj = os.lstat(obj) - except (OSError, AttributeError): - pass - islink = lstatobj is not None and stat.S_ISLNK(lstatobj.st_mode) - if lstatobj is None: - show_unmerge("---", unmerge_desc["!found"], file_type, obj) - continue - - f_match = obj[len(eroot)-1:] - ignore = False - for pattern in uninstall_ignore: - if fnmatch.fnmatch(f_match, pattern): - ignore = True - break - - if not ignore: - if islink and f_match in \ - ("/lib", "/usr/lib", "/usr/local/lib"): - # Ignore libdir symlinks for bug #423127. - ignore = True - - if ignore: - show_unmerge("---", unmerge_desc["cfgpro"], file_type, obj) - continue - - # don't use EROOT, CONTENTS entries already contain EPREFIX - if obj.startswith(real_root): - relative_path = obj[real_root_len:] - is_owned = False - for dblnk in others_in_slot: - if dblnk.isowner(relative_path): - is_owned = True - break - - if is_owned and islink and \ - file_type in ("sym", "dir") and \ - statobj and stat.S_ISDIR(statobj.st_mode): - # A new instance of this package claims the file, so - # don't unmerge it. If the file is symlink to a - # directory and the unmerging package installed it as - # a symlink, but the new owner has it listed as a - # directory, then we'll produce a warning since the - # symlink is a sort of orphan in this case (see - # bug #326685). - symlink_orphan = False - for dblnk in others_in_slot: - parent_contents_key = \ - dblnk._match_contents(relative_path) - if not parent_contents_key: - continue - if not parent_contents_key.startswith( - real_root): - continue - if dblnk.getcontents()[ - parent_contents_key][0] == "dir": - symlink_orphan = True - break - - if symlink_orphan: - protected_symlinks.setdefault( - (statobj.st_dev, statobj.st_ino), - []).append(relative_path) - - if is_owned: - show_unmerge("---", unmerge_desc["replaced"], file_type, obj) - continue - elif relative_path in cfgfiledict: - stale_confmem.append(relative_path) - - # Don't unlink symlinks to directories here since that can - # remove /lib and /usr/lib symlinks. - if unmerge_orphans and \ - lstatobj and not stat.S_ISDIR(lstatobj.st_mode) and \ - not (islink and statobj and stat.S_ISDIR(statobj.st_mode)) and \ - not self.isprotected(obj): - try: - unlink(obj, lstatobj) - except EnvironmentError as e: - if e.errno not in ignored_unlink_errnos: - raise - del e - show_unmerge("<<<", "", file_type, obj) - continue - - lmtime = str(lstatobj[stat.ST_MTIME]) - if (pkgfiles[objkey][0] not in ("dir", "fif", "dev")) and (lmtime != pkgfiles[objkey][1]): - show_unmerge("---", unmerge_desc["!mtime"], file_type, obj) - continue - - if file_type == "dir" and not islink: - if lstatobj is None or not stat.S_ISDIR(lstatobj.st_mode): - show_unmerge("---", unmerge_desc["!dir"], file_type, obj) - continue - mydirs.add((obj, (lstatobj.st_dev, lstatobj.st_ino))) - elif file_type == "sym" or (file_type == "dir" and islink): - if not islink: - show_unmerge("---", unmerge_desc["!sym"], file_type, obj) - continue - - # If this symlink points to a directory then we don't want - # to unmerge it if there are any other packages that - # installed files into the directory via this symlink - # (see bug #326685). - # TODO: Resolving a symlink to a directory will require - # simulation if $ROOT != / and the link is not relative. - if islink and statobj and stat.S_ISDIR(statobj.st_mode) \ - and obj.startswith(real_root): - - relative_path = obj[real_root_len:] - try: - target_dir_contents = os.listdir(obj) - except OSError: - pass - else: - if target_dir_contents: - # If all the children are regular files owned - # by this package, then the symlink should be - # safe to unmerge. - all_owned = True - for child in target_dir_contents: - child = os.path.join(relative_path, child) - if not self.isowner(child): - all_owned = False - break - try: - child_lstat = os.lstat(os.path.join( - real_root, child.lstrip(os.sep))) - except OSError: - continue - - if not stat.S_ISREG(child_lstat.st_mode): - # Nested symlinks or directories make - # the issue very complex, so just - # preserve the symlink in order to be - # on the safe side. - all_owned = False - break - - if not all_owned: - protected_symlinks.setdefault( - (statobj.st_dev, statobj.st_ino), - []).append(relative_path) - show_unmerge("---", unmerge_desc["!empty"], - file_type, obj) - continue - - # Go ahead and unlink symlinks to directories here when - # they're actually recorded as symlinks in the contents. - # Normally, symlinks such as /lib -> lib64 are not recorded - # as symlinks in the contents of a package. If a package - # installs something into ${D}/lib/, it is recorded in the - # contents as a directory even if it happens to correspond - # to a symlink when it's merged to the live filesystem. - try: - unlink(obj, lstatobj) - show_unmerge("<<<", "", file_type, obj) - except (OSError, IOError) as e: - if e.errno not in ignored_unlink_errnos: - raise - del e - show_unmerge("!!!", "", file_type, obj) - elif pkgfiles[objkey][0] == "obj": - if statobj is None or not stat.S_ISREG(statobj.st_mode): - show_unmerge("---", unmerge_desc["!obj"], file_type, obj) - continue - mymd5 = None - try: - mymd5 = perf_md5(obj, calc_prelink=calc_prelink) - except FileNotFound as e: - # the file has disappeared between now and our stat call - show_unmerge("---", unmerge_desc["!obj"], file_type, obj) - continue - - # string.lower is needed because db entries used to be in upper-case. The - # string.lower allows for backwards compatibility. - if mymd5 != pkgfiles[objkey][2].lower(): - show_unmerge("---", unmerge_desc["!md5"], file_type, obj) - continue - try: - unlink(obj, lstatobj) - except (OSError, IOError) as e: - if e.errno not in ignored_unlink_errnos: - raise - del e - show_unmerge("<<<", "", file_type, obj) - elif pkgfiles[objkey][0] == "fif": - if not stat.S_ISFIFO(lstatobj[stat.ST_MODE]): - show_unmerge("---", unmerge_desc["!fif"], file_type, obj) - continue - show_unmerge("---", "", file_type, obj) - elif pkgfiles[objkey][0] == "dev": - show_unmerge("---", "", file_type, obj) - - self._unmerge_dirs(mydirs, infodirs_inodes, - protected_symlinks, unmerge_desc, unlink, os) - mydirs.clear() - - if protected_symlinks: - self._unmerge_protected_symlinks(others_in_slot, infodirs_inodes, - protected_symlinks, unmerge_desc, unlink, os) - - if protected_symlinks: - msg = "One or more symlinks to directories have been " + \ - "preserved in order to ensure that files installed " + \ - "via these symlinks remain accessible. " + \ - "This indicates that the mentioned symlink(s) may " + \ - "be obsolete remnants of an old install, and it " + \ - "may be appropriate to replace a given symlink " + \ - "with the directory that it points to." - lines = textwrap.wrap(msg, 72) - lines.append("") - flat_list = set() - flat_list.update(*protected_symlinks.values()) - flat_list = sorted(flat_list) - for f in flat_list: - lines.append("\t%s" % (os.path.join(real_root, - f.lstrip(os.sep)))) - lines.append("") - self._elog("elog", "postrm", lines) - - # Remove stale entries from config memory. - if stale_confmem: - for filename in stale_confmem: - del cfgfiledict[filename] - writedict(cfgfiledict, self.vartree.dbapi._conf_mem_file) - - #remove self from vartree database so that our own virtual gets zapped if we're the last node - self.vartree.zap(self.mycpv) - - def _unmerge_protected_symlinks(self, others_in_slot, infodirs_inodes, - protected_symlinks, unmerge_desc, unlink, os): - - real_root = self.settings['ROOT'] - show_unmerge = self._show_unmerge - ignored_unlink_errnos = self._ignored_unlink_errnos - - flat_list = set() - flat_list.update(*protected_symlinks.values()) - flat_list = sorted(flat_list) - - for f in flat_list: - for dblnk in others_in_slot: - if dblnk.isowner(f): - # If another package in the same slot installed - # a file via a protected symlink, return early - # and don't bother searching for any other owners. - return - - msg = [] - msg.append("") - msg.append(_("Directory symlink(s) may need protection:")) - msg.append("") - - for f in flat_list: - msg.append("\t%s" % \ - os.path.join(real_root, f.lstrip(os.path.sep))) - - msg.append("") - msg.append("Use the UNINSTALL_IGNORE variable to exempt specific symlinks") - msg.append("from the following search (see the make.conf man page).") - msg.append("") - msg.append(_("Searching all installed" - " packages for files installed via above symlink(s)...")) - msg.append("") - self._elog("elog", "postrm", msg) - - self.lockdb() - try: - owners = self.vartree.dbapi._owners.get_owners(flat_list) - self.vartree.dbapi.flush_cache() - finally: - self.unlockdb() - - for owner in list(owners): - if owner.mycpv == self.mycpv: - owners.pop(owner, None) - - if not owners: - msg = [] - msg.append(_("The above directory symlink(s) are all " - "safe to remove. Removing them now...")) - msg.append("") - self._elog("elog", "postrm", msg) - dirs = set() - for unmerge_syms in protected_symlinks.values(): - for relative_path in unmerge_syms: - obj = os.path.join(real_root, - relative_path.lstrip(os.sep)) - parent = os.path.dirname(obj) - while len(parent) > len(self._eroot): - try: - lstatobj = os.lstat(parent) - except OSError: - break - else: - dirs.add((parent, - (lstatobj.st_dev, lstatobj.st_ino))) - parent = os.path.dirname(parent) - try: - unlink(obj, os.lstat(obj)) - show_unmerge("<<<", "", "sym", obj) - except (OSError, IOError) as e: - if e.errno not in ignored_unlink_errnos: - raise - del e - show_unmerge("!!!", "", "sym", obj) - - protected_symlinks.clear() - self._unmerge_dirs(dirs, infodirs_inodes, - protected_symlinks, unmerge_desc, unlink, os) - dirs.clear() - - def _unmerge_dirs(self, dirs, infodirs_inodes, - protected_symlinks, unmerge_desc, unlink, os): - - show_unmerge = self._show_unmerge - infodir_cleanup = self._infodir_cleanup - ignored_unlink_errnos = self._ignored_unlink_errnos - ignored_rmdir_errnos = self._ignored_rmdir_errnos - real_root = self.settings['ROOT'] - - dirs = sorted(dirs) - revisit = {} - - while True: - try: - obj, inode_key = dirs.pop() - except IndexError: - break - # Treat any directory named "info" as a candidate here, - # since it might have been in INFOPATH previously even - # though it may not be there now. - if inode_key in infodirs_inodes or \ - os.path.basename(obj) == "info": - try: - remaining = os.listdir(obj) - except OSError: - pass - else: - cleanup_info_dir = () - if remaining and \ - len(remaining) <= len(infodir_cleanup): - if not set(remaining).difference(infodir_cleanup): - cleanup_info_dir = remaining - - for child in cleanup_info_dir: - child = os.path.join(obj, child) - try: - lstatobj = os.lstat(child) - if stat.S_ISREG(lstatobj.st_mode): - unlink(child, lstatobj) - show_unmerge("<<<", "", "obj", child) - except EnvironmentError as e: - if e.errno not in ignored_unlink_errnos: - raise - del e - show_unmerge("!!!", "", "obj", child) - - try: - parent_name = os.path.dirname(obj) - parent_stat = os.stat(parent_name) - - if bsd_chflags: - lstatobj = os.lstat(obj) - if lstatobj.st_flags != 0: - bsd_chflags.lchflags(obj, 0) - - # Use normal stat/chflags for the parent since we want to - # follow any symlinks to the real parent directory. - pflags = parent_stat.st_flags - if pflags != 0: - bsd_chflags.chflags(parent_name, 0) - try: - os.rmdir(obj) - finally: - if bsd_chflags and pflags != 0: - # Restore the parent flags we saved before unlinking - bsd_chflags.chflags(parent_name, pflags) - - # Record the parent directory for use in syncfs calls. - # Note that we use a realpath and a regular stat here, since - # we want to follow any symlinks back to the real device where - # the real parent directory resides. - self._merged_path(os.path.realpath(parent_name), parent_stat) - - show_unmerge("<<<", "", "dir", obj) - except EnvironmentError as e: - if e.errno not in ignored_rmdir_errnos: - raise - if e.errno != errno.ENOENT: - show_unmerge("---", unmerge_desc["!empty"], "dir", obj) - revisit[obj] = inode_key - - # Since we didn't remove this directory, record the directory - # itself for use in syncfs calls, if we have removed another - # file from the same device. - # Note that we use a realpath and a regular stat here, since - # we want to follow any symlinks back to the real device where - # the real directory resides. - try: - dir_stat = os.stat(obj) - except OSError: - pass - else: - if dir_stat.st_dev in self._device_path_map: - self._merged_path(os.path.realpath(obj), dir_stat) - - else: - # When a directory is successfully removed, there's - # no need to protect symlinks that point to it. - unmerge_syms = protected_symlinks.pop(inode_key, None) - if unmerge_syms is not None: - parents = [] - for relative_path in unmerge_syms: - obj = os.path.join(real_root, - relative_path.lstrip(os.sep)) - try: - unlink(obj, os.lstat(obj)) - show_unmerge("<<<", "", "sym", obj) - except (OSError, IOError) as e: - if e.errno not in ignored_unlink_errnos: - raise - del e - show_unmerge("!!!", "", "sym", obj) - else: - parents.append(os.path.dirname(obj)) - - if parents: - # Revisit parents recursively (bug 640058). - recursive_parents = [] - for parent in set(parents): - while parent in revisit: - recursive_parents.append(parent) - parent = os.path.dirname(parent) - if parent == '/': - break - - for parent in sorted(set(recursive_parents)): - dirs.append((parent, revisit.pop(parent))) - - def isowner(self, filename, destroot=None): - """ - Check if a file belongs to this package. This may - result in a stat call for the parent directory of - every installed file, since the inode numbers are - used to work around the problem of ambiguous paths - caused by symlinked directories. The results of - stat calls are cached to optimize multiple calls - to this method. - - @param filename: - @type filename: - @param destroot: - @type destroot: - @rtype: Boolean - @return: - 1. True if this package owns the file. - 2. False if this package does not own the file. - """ - - if destroot is not None and destroot != self._eroot: - warnings.warn("The second parameter of the " + \ - "portage.dbapi.vartree.dblink.isowner()" + \ - " is now unused. Instead " + \ - "self.settings['EROOT'] will be used.", - DeprecationWarning, stacklevel=2) - - return bool(self._match_contents(filename)) - - def _match_contents(self, filename, destroot=None): - """ - The matching contents entry is returned, which is useful - since the path may differ from the one given by the caller, - due to symlinks. - - @rtype: String - @return: the contents entry corresponding to the given path, or False - if the file is not owned by this package. - """ - - filename = _unicode_decode(filename, - encoding=_encodings['content'], errors='strict') - - if destroot is not None and destroot != self._eroot: - warnings.warn("The second parameter of the " + \ - "portage.dbapi.vartree.dblink._match_contents()" + \ - " is now unused. Instead " + \ - "self.settings['ROOT'] will be used.", - DeprecationWarning, stacklevel=2) - - # don't use EROOT here, image already contains EPREFIX - destroot = self.settings['ROOT'] - - # The given filename argument might have a different encoding than the - # the filenames contained in the contents, so use separate wrapped os - # modules for each. The basename is more likely to contain non-ascii - # characters than the directory path, so use os_filename_arg for all - # operations involving the basename of the filename arg. - os_filename_arg = _os_merge - os = _os_merge - - try: - _unicode_encode(filename, - encoding=_encodings['merge'], errors='strict') - except UnicodeEncodeError: - # The package appears to have been merged with a - # different value of sys.getfilesystemencoding(), - # so fall back to utf_8 if appropriate. - try: - _unicode_encode(filename, - encoding=_encodings['fs'], errors='strict') - except UnicodeEncodeError: - pass - else: - os_filename_arg = portage.os - - destfile = normalize_path( - os_filename_arg.path.join(destroot, - filename.lstrip(os_filename_arg.path.sep))) - - if "case-insensitive-fs" in self.settings.features: - destfile = destfile.lower() - - if self._contents.contains(destfile): - return self._contents.unmap_key(destfile) - - if self.getcontents(): - basename = os_filename_arg.path.basename(destfile) - if self._contents_basenames is None: - - try: - for x in self._contents.keys(): - _unicode_encode(x, - encoding=_encodings['merge'], - errors='strict') - except UnicodeEncodeError: - # The package appears to have been merged with a - # different value of sys.getfilesystemencoding(), - # so fall back to utf_8 if appropriate. - try: - for x in self._contents.keys(): - _unicode_encode(x, - encoding=_encodings['fs'], - errors='strict') - except UnicodeEncodeError: - pass - else: - os = portage.os - - self._contents_basenames = set( - os.path.basename(x) for x in self._contents.keys()) - if basename not in self._contents_basenames: - # This is a shortcut that, in most cases, allows us to - # eliminate this package as an owner without the need - # to examine inode numbers of parent directories. - return False - - # Use stat rather than lstat since we want to follow - # any symlinks to the real parent directory. - parent_path = os_filename_arg.path.dirname(destfile) - try: - parent_stat = os_filename_arg.stat(parent_path) - except EnvironmentError as e: - if e.errno != errno.ENOENT: - raise - del e - return False - if self._contents_inodes is None: - - if os is _os_merge: - try: - for x in self._contents.keys(): - _unicode_encode(x, - encoding=_encodings['merge'], - errors='strict') - except UnicodeEncodeError: - # The package appears to have been merged with a - # different value of sys.getfilesystemencoding(), - # so fall back to utf_8 if appropriate. - try: - for x in self._contents.keys(): - _unicode_encode(x, - encoding=_encodings['fs'], - errors='strict') - except UnicodeEncodeError: - pass - else: - os = portage.os - - self._contents_inodes = {} - parent_paths = set() - for x in self._contents.keys(): - p_path = os.path.dirname(x) - if p_path in parent_paths: - continue - parent_paths.add(p_path) - try: - s = os.stat(p_path) - except OSError: - pass - else: - inode_key = (s.st_dev, s.st_ino) - # Use lists of paths in case multiple - # paths reference the same inode. - p_path_list = self._contents_inodes.get(inode_key) - if p_path_list is None: - p_path_list = [] - self._contents_inodes[inode_key] = p_path_list - if p_path not in p_path_list: - p_path_list.append(p_path) - - p_path_list = self._contents_inodes.get( - (parent_stat.st_dev, parent_stat.st_ino)) - if p_path_list: - for p_path in p_path_list: - x = os_filename_arg.path.join(p_path, basename) - if self._contents.contains(x): - return self._contents.unmap_key(x) - - return False - - def _linkmap_rebuild(self, **kwargs): - """ - Rebuild the self._linkmap if it's not broken due to missing - scanelf binary. Also, return early if preserve-libs is disabled - and the preserve-libs registry is empty. - """ - if self._linkmap_broken or \ - self.vartree.dbapi._linkmap is None or \ - self.vartree.dbapi._plib_registry is None or \ - ("preserve-libs" not in self.settings.features and \ - not self.vartree.dbapi._plib_registry.hasEntries()): - return - try: - self.vartree.dbapi._linkmap.rebuild(**kwargs) - except CommandNotFound as e: - self._linkmap_broken = True - self._display_merge(_("!!! Disabling preserve-libs " \ - "due to error: Command Not Found: %s\n") % (e,), - level=logging.ERROR, noiselevel=-1) - - def _find_libs_to_preserve(self, unmerge=False): - """ - Get set of relative paths for libraries to be preserved. When - unmerge is False, file paths to preserve are selected from - self._installed_instance. Otherwise, paths are selected from - self. - """ - if self._linkmap_broken or \ - self.vartree.dbapi._linkmap is None or \ - self.vartree.dbapi._plib_registry is None or \ - (not unmerge and self._installed_instance is None) or \ - not self._preserve_libs: - return set() - - os = _os_merge - linkmap = self.vartree.dbapi._linkmap - if unmerge: - installed_instance = self - else: - installed_instance = self._installed_instance - old_contents = installed_instance.getcontents() - root = self.settings['ROOT'] - root_len = len(root) - 1 - lib_graph = digraph() - path_node_map = {} - - def path_to_node(path): - node = path_node_map.get(path) - if node is None: - node = LinkageMap._LibGraphNode(linkmap._obj_key(path)) - alt_path_node = lib_graph.get(node) - if alt_path_node is not None: - node = alt_path_node - node.alt_paths.add(path) - path_node_map[path] = node - return node - - consumer_map = {} - provider_nodes = set() - # Create provider nodes and add them to the graph. - for f_abs in old_contents: - - if os is _os_merge: - try: - _unicode_encode(f_abs, - encoding=_encodings['merge'], errors='strict') - except UnicodeEncodeError: - # The package appears to have been merged with a - # different value of sys.getfilesystemencoding(), - # so fall back to utf_8 if appropriate. - try: - _unicode_encode(f_abs, - encoding=_encodings['fs'], errors='strict') - except UnicodeEncodeError: - pass - else: - os = portage.os - - f = f_abs[root_len:] - if not unmerge and self.isowner(f): - # We have an indentically named replacement file, - # so we don't try to preserve the old copy. - continue - try: - consumers = linkmap.findConsumers(f, - exclude_providers=(installed_instance.isowner,)) - except KeyError: - continue - if not consumers: - continue - provider_node = path_to_node(f) - lib_graph.add(provider_node, None) - provider_nodes.add(provider_node) - consumer_map[provider_node] = consumers - - # Create consumer nodes and add them to the graph. - # Note that consumers can also be providers. - for provider_node, consumers in consumer_map.items(): - for c in consumers: - consumer_node = path_to_node(c) - if installed_instance.isowner(c) and \ - consumer_node not in provider_nodes: - # This is not a provider, so it will be uninstalled. - continue - lib_graph.add(provider_node, consumer_node) - - # Locate nodes which should be preserved. They consist of all - # providers that are reachable from consumers that are not - # providers themselves. - preserve_nodes = set() - for consumer_node in lib_graph.root_nodes(): - if consumer_node in provider_nodes: - continue - # Preserve all providers that are reachable from this consumer. - node_stack = lib_graph.child_nodes(consumer_node) - while node_stack: - provider_node = node_stack.pop() - if provider_node in preserve_nodes: - continue - preserve_nodes.add(provider_node) - node_stack.extend(lib_graph.child_nodes(provider_node)) - - preserve_paths = set() - for preserve_node in preserve_nodes: - # Preserve the library itself, and also preserve the - # soname symlink which is the only symlink that is - # strictly required. - hardlinks = set() - soname_symlinks = set() - soname = linkmap.getSoname(next(iter(preserve_node.alt_paths))) - for f in preserve_node.alt_paths: - f_abs = os.path.join(root, f.lstrip(os.sep)) - try: - if stat.S_ISREG(os.lstat(f_abs).st_mode): - hardlinks.add(f) - elif os.path.basename(f) == soname: - soname_symlinks.add(f) - except OSError: - pass - - if hardlinks: - preserve_paths.update(hardlinks) - preserve_paths.update(soname_symlinks) - - return preserve_paths - - def _add_preserve_libs_to_contents(self, preserve_paths): - """ - Preserve libs returned from _find_libs_to_preserve(). - """ - - if not preserve_paths: - return - - os = _os_merge - showMessage = self._display_merge - root = self.settings['ROOT'] - - # Copy contents entries from the old package to the new one. - new_contents = self.getcontents().copy() - old_contents = self._installed_instance.getcontents() - for f in sorted(preserve_paths): - f = _unicode_decode(f, - encoding=_encodings['content'], errors='strict') - f_abs = os.path.join(root, f.lstrip(os.sep)) - contents_entry = old_contents.get(f_abs) - if contents_entry is None: - # This will probably never happen, but it might if one of the - # paths returned from findConsumers() refers to one of the libs - # that should be preserved yet the path is not listed in the - # contents. Such a path might belong to some other package, so - # it shouldn't be preserved here. - showMessage(_("!!! File '%s' will not be preserved " - "due to missing contents entry\n") % (f_abs,), - level=logging.ERROR, noiselevel=-1) - preserve_paths.remove(f) - continue - new_contents[f_abs] = contents_entry - obj_type = contents_entry[0] - showMessage(_(">>> needed %s %s\n") % (obj_type, f_abs), - noiselevel=-1) - # Add parent directories to contents if necessary. - parent_dir = os.path.dirname(f_abs) - while len(parent_dir) > len(root): - new_contents[parent_dir] = ["dir"] - prev = parent_dir - parent_dir = os.path.dirname(parent_dir) - if prev == parent_dir: - break - outfile = atomic_ofstream(os.path.join(self.dbtmpdir, "CONTENTS")) - write_contents(new_contents, root, outfile) - outfile.close() - self._clear_contents_cache() - - def _find_unused_preserved_libs(self, unmerge_no_replacement): - """ - Find preserved libraries that don't have any consumers left. - """ - - if self._linkmap_broken or \ - self.vartree.dbapi._linkmap is None or \ - self.vartree.dbapi._plib_registry is None or \ - not self.vartree.dbapi._plib_registry.hasEntries(): - return {} - - # Since preserved libraries can be consumers of other preserved - # libraries, use a graph to track consumer relationships. - plib_dict = self.vartree.dbapi._plib_registry.getPreservedLibs() - linkmap = self.vartree.dbapi._linkmap - lib_graph = digraph() - preserved_nodes = set() - preserved_paths = set() - path_cpv_map = {} - path_node_map = {} - root = self.settings['ROOT'] - - def path_to_node(path): - node = path_node_map.get(path) - if node is None: - node = LinkageMap._LibGraphNode(linkmap._obj_key(path)) - alt_path_node = lib_graph.get(node) - if alt_path_node is not None: - node = alt_path_node - node.alt_paths.add(path) - path_node_map[path] = node - return node - - for cpv, plibs in plib_dict.items(): - for f in plibs: - path_cpv_map[f] = cpv - preserved_node = path_to_node(f) - if not preserved_node.file_exists(): - continue - lib_graph.add(preserved_node, None) - preserved_paths.add(f) - preserved_nodes.add(preserved_node) - for c in self.vartree.dbapi._linkmap.findConsumers(f): - consumer_node = path_to_node(c) - if not consumer_node.file_exists(): - continue - # Note that consumers may also be providers. - lib_graph.add(preserved_node, consumer_node) - - # Eliminate consumers having providers with the same soname as an - # installed library that is not preserved. This eliminates - # libraries that are erroneously preserved due to a move from one - # directory to another. - # Also eliminate consumers that are going to be unmerged if - # unmerge_no_replacement is True. - provider_cache = {} - for preserved_node in preserved_nodes: - soname = linkmap.getSoname(preserved_node) - for consumer_node in lib_graph.parent_nodes(preserved_node): - if consumer_node in preserved_nodes: - continue - if unmerge_no_replacement: - will_be_unmerged = True - for path in consumer_node.alt_paths: - if not self.isowner(path): - will_be_unmerged = False - break - if will_be_unmerged: - # This consumer is not preserved and it is - # being unmerged, so drop this edge. - lib_graph.remove_edge(preserved_node, consumer_node) - continue - - providers = provider_cache.get(consumer_node) - if providers is None: - providers = linkmap.findProviders(consumer_node) - provider_cache[consumer_node] = providers - providers = providers.get(soname) - if providers is None: - continue - for provider in providers: - if provider in preserved_paths: - continue - provider_node = path_to_node(provider) - if not provider_node.file_exists(): - continue - if provider_node in preserved_nodes: - continue - # An alternative provider seems to be - # installed, so drop this edge. - lib_graph.remove_edge(preserved_node, consumer_node) - break - - cpv_lib_map = {} - while lib_graph: - root_nodes = preserved_nodes.intersection(lib_graph.root_nodes()) - if not root_nodes: - break - lib_graph.difference_update(root_nodes) - unlink_list = set() - for node in root_nodes: - unlink_list.update(node.alt_paths) - unlink_list = sorted(unlink_list) - for obj in unlink_list: - cpv = path_cpv_map.get(obj) - if cpv is None: - # This means that a symlink is in the preserved libs - # registry, but the actual lib it points to is not. - self._display_merge(_("!!! symlink to lib is preserved, " - "but not the lib itself:\n!!! '%s'\n") % (obj,), - level=logging.ERROR, noiselevel=-1) - continue - removed = cpv_lib_map.get(cpv) - if removed is None: - removed = set() - cpv_lib_map[cpv] = removed - removed.add(obj) - - return cpv_lib_map - - def _remove_preserved_libs(self, cpv_lib_map): - """ - Remove files returned from _find_unused_preserved_libs(). - """ - - os = _os_merge - - files_to_remove = set() - for files in cpv_lib_map.values(): - files_to_remove.update(files) - files_to_remove = sorted(files_to_remove) - showMessage = self._display_merge - root = self.settings['ROOT'] - - parent_dirs = set() - for obj in files_to_remove: - obj = os.path.join(root, obj.lstrip(os.sep)) - parent_dirs.add(os.path.dirname(obj)) - if os.path.islink(obj): - obj_type = _("sym") - else: - obj_type = _("obj") - try: - os.unlink(obj) - except OSError as e: - if e.errno != errno.ENOENT: - raise - del e - else: - showMessage(_("<<< !needed %s %s\n") % (obj_type, obj), - noiselevel=-1) - - # Remove empty parent directories if possible. - while parent_dirs: - x = parent_dirs.pop() - while True: - try: - os.rmdir(x) - except OSError: - break - prev = x - x = os.path.dirname(x) - if x == prev: - break - - self.vartree.dbapi._plib_registry.pruneNonExisting() - - def _collision_protect(self, srcroot, destroot, mypkglist, - file_list, symlink_list): - - os = _os_merge - - collision_ignore = [] - for x in portage.util.shlex_split( - self.settings.get("COLLISION_IGNORE", "")): - if os.path.isdir(os.path.join(self._eroot, x.lstrip(os.sep))): - x = normalize_path(x) - x += "/*" - collision_ignore.append(x) - - # For collisions with preserved libraries, the current package - # will assume ownership and the libraries will be unregistered. - if self.vartree.dbapi._plib_registry is None: - # preserve-libs is entirely disabled - plib_cpv_map = None - plib_paths = None - plib_inodes = {} - else: - plib_dict = self.vartree.dbapi._plib_registry.getPreservedLibs() - plib_cpv_map = {} - plib_paths = set() - for cpv, paths in plib_dict.items(): - plib_paths.update(paths) - for f in paths: - plib_cpv_map[f] = cpv - plib_inodes = self._lstat_inode_map(plib_paths) - - plib_collisions = {} - - showMessage = self._display_merge - stopmerge = False - collisions = [] - dirs = set() - dirs_ro = set() - symlink_collisions = [] - destroot = self.settings['ROOT'] - totfiles = len(file_list) + len(symlink_list) - showMessage(_(" %s checking %d files for package collisions\n") % \ - (colorize("GOOD", "*"), totfiles)) - for i, (f, f_type) in enumerate(chain( - ((f, "reg") for f in file_list), - ((f, "sym") for f in symlink_list))): - if i % 1000 == 0 and i != 0: - showMessage(_("%d files remaining ...\n") % (totfiles - i)) - - dest_path = normalize_path( - os.path.join(destroot, f.lstrip(os.path.sep))) - - parent = os.path.dirname(dest_path) - if parent not in dirs: - for x in iter_parents(parent): - if x in dirs: - break - dirs.add(x) - if os.path.isdir(x): - if not os.access(x, os.W_OK): - dirs_ro.add(x) - break - - try: - dest_lstat = os.lstat(dest_path) - except EnvironmentError as e: - if e.errno == errno.ENOENT: - del e - continue - elif e.errno == errno.ENOTDIR: - del e - # A non-directory is in a location where this package - # expects to have a directory. - dest_lstat = None - parent_path = dest_path - while len(parent_path) > len(destroot): - parent_path = os.path.dirname(parent_path) - try: - dest_lstat = os.lstat(parent_path) - break - except EnvironmentError as e: - if e.errno != errno.ENOTDIR: - raise - del e - if not dest_lstat: - raise AssertionError( - "unable to find non-directory " + \ - "parent for '%s'" % dest_path) - dest_path = parent_path - f = os.path.sep + dest_path[len(destroot):] - if f in collisions: - continue - else: - raise - if f[0] != "/": - f="/"+f - - if stat.S_ISDIR(dest_lstat.st_mode): - if f_type == "sym": - # This case is explicitly banned - # by PMS (see bug #326685). - symlink_collisions.append(f) - collisions.append(f) - continue - - plibs = plib_inodes.get((dest_lstat.st_dev, dest_lstat.st_ino)) - if plibs: - for path in plibs: - cpv = plib_cpv_map[path] - paths = plib_collisions.get(cpv) - if paths is None: - paths = set() - plib_collisions[cpv] = paths - paths.add(path) - # The current package will assume ownership and the - # libraries will be unregistered, so exclude this - # path from the normal collisions. - continue - - isowned = False - full_path = os.path.join(destroot, f.lstrip(os.path.sep)) - for ver in mypkglist: - if ver.isowner(f): - isowned = True - break - if not isowned and self.isprotected(full_path): - isowned = True - if not isowned: - f_match = full_path[len(self._eroot)-1:] - stopmerge = True - for pattern in collision_ignore: - if fnmatch.fnmatch(f_match, pattern): - stopmerge = False - break - if stopmerge: - collisions.append(f) - return collisions, dirs_ro, symlink_collisions, plib_collisions - - def _lstat_inode_map(self, path_iter): - """ - Use lstat to create a map of the form: - {(st_dev, st_ino) : set([path1, path2, ...])} - Multiple paths may reference the same inode due to hardlinks. - All lstat() calls are relative to self.myroot. - """ - - os = _os_merge - - root = self.settings['ROOT'] - inode_map = {} - for f in path_iter: - path = os.path.join(root, f.lstrip(os.sep)) - try: - st = os.lstat(path) - except OSError as e: - if e.errno not in (errno.ENOENT, errno.ENOTDIR): - raise - del e - continue - key = (st.st_dev, st.st_ino) - paths = inode_map.get(key) - if paths is None: - paths = set() - inode_map[key] = paths - paths.add(f) - return inode_map - - def _security_check(self, installed_instances): - if not installed_instances: - return 0 - - os = _os_merge - - showMessage = self._display_merge - - file_paths = set() - for dblnk in installed_instances: - file_paths.update(dblnk.getcontents()) - inode_map = {} - real_paths = set() - for i, path in enumerate(file_paths): - - if os is _os_merge: - try: - _unicode_encode(path, - encoding=_encodings['merge'], errors='strict') - except UnicodeEncodeError: - # The package appears to have been merged with a - # different value of sys.getfilesystemencoding(), - # so fall back to utf_8 if appropriate. - try: - _unicode_encode(path, - encoding=_encodings['fs'], errors='strict') - except UnicodeEncodeError: - pass - else: - os = portage.os - - try: - s = os.lstat(path) - except OSError as e: - if e.errno not in (errno.ENOENT, errno.ENOTDIR): - raise - del e - continue - if not stat.S_ISREG(s.st_mode): - continue - path = os.path.realpath(path) - if path in real_paths: - continue - real_paths.add(path) - if s.st_nlink > 1 and \ - s.st_mode & (stat.S_ISUID | stat.S_ISGID): - k = (s.st_dev, s.st_ino) - inode_map.setdefault(k, []).append((path, s)) - suspicious_hardlinks = [] - for path_list in inode_map.values(): - path, s = path_list[0] - if len(path_list) == s.st_nlink: - # All hardlinks seem to be owned by this package. - continue - suspicious_hardlinks.append(path_list) - if not suspicious_hardlinks: - return 0 - - msg = [] - msg.append(_("suid/sgid file(s) " - "with suspicious hardlink(s):")) - msg.append("") - for path_list in suspicious_hardlinks: - for path, s in path_list: - msg.append("\t%s" % path) - msg.append("") - msg.append(_("See the Gentoo Security Handbook " - "guide for advice on how to proceed.")) - - self._eerror("preinst", msg) - - return 1 - - def _eqawarn(self, phase, lines): - self._elog("eqawarn", phase, lines) - - def _eerror(self, phase, lines): - self._elog("eerror", phase, lines) - - def _elog(self, funcname, phase, lines): - func = getattr(portage.elog.messages, funcname) - if self._scheduler is None: - for l in lines: - func(l, phase=phase, key=self.mycpv) - else: - background = self.settings.get("PORTAGE_BACKGROUND") == "1" - log_path = None - if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": - log_path = self.settings.get("PORTAGE_LOG_FILE") - out = io.StringIO() - for line in lines: - func(line, phase=phase, key=self.mycpv, out=out) - msg = out.getvalue() - self._scheduler.output(msg, - background=background, log_path=log_path) - - def _elog_process(self, phasefilter=None): - cpv = self.mycpv - if self._pipe is None: - elog_process(cpv, self.settings, phasefilter=phasefilter) - else: - logdir = os.path.join(self.settings["T"], "logging") - ebuild_logentries = collect_ebuild_messages(logdir) - # phasefilter is irrelevant for the above collect_ebuild_messages - # call, since this package instance has a private logdir. However, - # it may be relevant for the following collect_messages call. - py_logentries = collect_messages(key=cpv, phasefilter=phasefilter).get(cpv, {}) - logentries = _merge_logentries(py_logentries, ebuild_logentries) - funcnames = { - "INFO": "einfo", - "LOG": "elog", - "WARN": "ewarn", - "QA": "eqawarn", - "ERROR": "eerror" - } - str_buffer = [] - for phase, messages in logentries.items(): - for key, lines in messages: - funcname = funcnames[key] - if isinstance(lines, basestring): - lines = [lines] - for line in lines: - for line in line.split('\n'): - fields = (funcname, phase, cpv, line) - str_buffer.append(' '.join(fields)) - str_buffer.append('\n') - if str_buffer: - str_buffer = _unicode_encode(''.join(str_buffer)) - while str_buffer: - str_buffer = str_buffer[os.write(self._pipe, str_buffer):] - - def _emerge_log(self, msg): - emergelog(False, msg) - - def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=0, - mydbapi=None, prev_mtimes=None, counter=None): - """ - - This function does the following: - - calls get_ro_checker to retrieve a function for checking whether Portage - will write to a read-only filesystem, then runs it against the directory list - calls self._preserve_libs if FEATURES=preserve-libs - calls self._collision_protect if FEATURES=collision-protect - calls doebuild(mydo=pkg_preinst) - Merges the package to the livefs - unmerges old version (if required) - calls doebuild(mydo=pkg_postinst) - calls env_update - - @param srcroot: Typically this is ${D} - @type srcroot: String (Path) - @param destroot: ignored, self.settings['ROOT'] is used instead - @type destroot: String (Path) - @param inforoot: root of the vardb entry ? - @type inforoot: String (Path) - @param myebuild: path to the ebuild that we are processing - @type myebuild: String (Path) - @param mydbapi: dbapi which is handed to doebuild. - @type mydbapi: portdbapi instance - @param prev_mtimes: { Filename:mtime } mapping for env_update - @type prev_mtimes: Dictionary - @rtype: Boolean - @return: - 1. 0 on success - 2. 1 on failure - - secondhand is a list of symlinks that have been skipped due to their target - not existing; we will merge these symlinks at a later time. - """ - - os = _os_merge - - srcroot = _unicode_decode(srcroot, - encoding=_encodings['content'], errors='strict') - destroot = self.settings['ROOT'] - inforoot = _unicode_decode(inforoot, - encoding=_encodings['content'], errors='strict') - myebuild = _unicode_decode(myebuild, - encoding=_encodings['content'], errors='strict') - - showMessage = self._display_merge - srcroot = normalize_path(srcroot).rstrip(os.path.sep) + os.path.sep - - if not os.path.isdir(srcroot): - showMessage(_("!!! Directory Not Found: D='%s'\n") % srcroot, - level=logging.ERROR, noiselevel=-1) - return 1 - - is_binpkg = self.settings.get("EMERGE_FROM") == "binary" - slot = '' - for var_name in ('CHOST', 'SLOT'): - try: - with io.open(_unicode_encode( - os.path.join(inforoot, var_name), - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], - errors='replace') as f: - val = f.readline().strip() - except EnvironmentError as e: - if e.errno != errno.ENOENT: - raise - del e - val = '' - - if var_name == 'SLOT': - slot = val - - if not slot.strip(): - slot = self.settings.get(var_name, '') - if not slot.strip(): - showMessage(_("!!! SLOT is undefined\n"), - level=logging.ERROR, noiselevel=-1) - return 1 - write_atomic(os.path.join(inforoot, var_name), slot + '\n') - - # This check only applies when built from source, since - # inforoot values are written just after src_install. - if not is_binpkg and val != self.settings.get(var_name, ''): - self._eqawarn('preinst', - [_("QA Notice: Expected %(var_name)s='%(expected_value)s', got '%(actual_value)s'\n") % \ - {"var_name":var_name, "expected_value":self.settings.get(var_name, ''), "actual_value":val}]) - - def eerror(lines): - self._eerror("preinst", lines) - - if not os.path.exists(self.dbcatdir): - ensure_dirs(self.dbcatdir) - - # NOTE: We use SLOT obtained from the inforoot - # directory, in order to support USE=multislot. - # Use _pkg_str discard the sub-slot part if necessary. - slot = _pkg_str(self.mycpv, slot=slot).slot - cp = self.mysplit[0] - slot_atom = "%s:%s" % (cp, slot) - - self.lockdb() - try: - # filter any old-style virtual matches - slot_matches = [cpv for cpv in self.vartree.dbapi.match(slot_atom) - if cpv_getkey(cpv) == cp] - - if self.mycpv not in slot_matches and \ - self.vartree.dbapi.cpv_exists(self.mycpv): - # handle multislot or unapplied slotmove - slot_matches.append(self.mycpv) - - others_in_slot = [] - for cur_cpv in slot_matches: - # Clone the config in case one of these has to be unmerged, - # since we need it to have private ${T} etc... for things - # like elog. - settings_clone = portage.config(clone=self.settings) - settings_clone.pop("PORTAGE_BUILDDIR_LOCKED", None) - settings_clone.setcpv(cur_cpv, mydb=self.vartree.dbapi) - if self._preserve_libs and "preserve-libs" in \ - settings_clone["PORTAGE_RESTRICT"].split(): - self._preserve_libs = False - others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1], - settings=settings_clone, - vartree=self.vartree, treetype="vartree", - scheduler=self._scheduler, pipe=self._pipe)) - finally: - self.unlockdb() - - # If any instance has RESTRICT=preserve-libs, then - # restrict it for all instances. - if not self._preserve_libs: - for dblnk in others_in_slot: - dblnk._preserve_libs = False - - retval = self._security_check(others_in_slot) - if retval: - return retval - - if slot_matches: - # Used by self.isprotected(). - max_dblnk = None - max_counter = -1 - for dblnk in others_in_slot: - cur_counter = self.vartree.dbapi.cpv_counter(dblnk.mycpv) - if cur_counter > max_counter: - max_counter = cur_counter - max_dblnk = dblnk - self._installed_instance = max_dblnk - - # Apply INSTALL_MASK before collision-protect, since it may - # be useful to avoid collisions in some scenarios. - # We cannot detect if this is needed or not here as INSTALL_MASK can be - # modified by bashrc files. - phase = MiscFunctionsProcess(background=False, - commands=["preinst_mask"], phase="preinst", - scheduler=self._scheduler, settings=self.settings) - phase.start() - phase.wait() - try: - with io.open(_unicode_encode(os.path.join(inforoot, "INSTALL_MASK"), - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], - errors='replace') as f: - install_mask = InstallMask(f.read()) - except EnvironmentError: - install_mask = None - - if install_mask: - install_mask_dir(self.settings["ED"], install_mask) - if any(x in self.settings.features for x in ('nodoc', 'noman', 'noinfo')): - try: - os.rmdir(os.path.join(self.settings["ED"], 'usr', 'share')) - except OSError: - pass - - # We check for unicode encoding issues after src_install. However, - # the check must be repeated here for binary packages (it's - # inexpensive since we call os.walk() here anyway). - unicode_errors = [] - line_ending_re = re.compile('[\n\r]') - srcroot_len = len(srcroot) - ed_len = len(self.settings["ED"]) - eprefix_len = len(self.settings["EPREFIX"]) - - while True: - - unicode_error = False - eagain_error = False - - filelist = [] - linklist = [] - paths_with_newlines = [] - def onerror(e): - raise - walk_iter = os.walk(srcroot, onerror=onerror) - while True: - try: - parent, dirs, files = next(walk_iter) - except StopIteration: - break - except OSError as e: - if e.errno != errno.EAGAIN: - raise - # Observed with PyPy 1.8. - eagain_error = True - break - - try: - parent = _unicode_decode(parent, - encoding=_encodings['merge'], errors='strict') - except UnicodeDecodeError: - new_parent = _unicode_decode(parent, - encoding=_encodings['merge'], errors='replace') - new_parent = _unicode_encode(new_parent, - encoding='ascii', errors='backslashreplace') - new_parent = _unicode_decode(new_parent, - encoding=_encodings['merge'], errors='replace') - os.rename(parent, new_parent) - unicode_error = True - unicode_errors.append(new_parent[ed_len:]) - break - - for fname in files: - try: - fname = _unicode_decode(fname, - encoding=_encodings['merge'], errors='strict') - except UnicodeDecodeError: - fpath = portage._os.path.join( - parent.encode(_encodings['merge']), fname) - new_fname = _unicode_decode(fname, - encoding=_encodings['merge'], errors='replace') - new_fname = _unicode_encode(new_fname, - encoding='ascii', errors='backslashreplace') - new_fname = _unicode_decode(new_fname, - encoding=_encodings['merge'], errors='replace') - new_fpath = os.path.join(parent, new_fname) - os.rename(fpath, new_fpath) - unicode_error = True - unicode_errors.append(new_fpath[ed_len:]) - fname = new_fname - fpath = new_fpath - else: - fpath = os.path.join(parent, fname) - - relative_path = fpath[srcroot_len:] - - if line_ending_re.search(relative_path) is not None: - paths_with_newlines.append(relative_path) - - file_mode = os.lstat(fpath).st_mode - if stat.S_ISREG(file_mode): - filelist.append(relative_path) - elif stat.S_ISLNK(file_mode): - # Note: os.walk puts symlinks to directories in the "dirs" - # list and it does not traverse them since that could lead - # to an infinite recursion loop. - linklist.append(relative_path) - - myto = _unicode_decode( - _os.readlink(_unicode_encode(fpath, - encoding=_encodings['merge'], errors='strict')), - encoding=_encodings['merge'], errors='replace') - if line_ending_re.search(myto) is not None: - paths_with_newlines.append(relative_path) - - if unicode_error: - break - - if not (unicode_error or eagain_error): - break - - if unicode_errors: - self._elog("eqawarn", "preinst", - _merge_unicode_error(unicode_errors)) - - if paths_with_newlines: - msg = [] - msg.append(_("This package installs one or more files containing line ending characters:")) - msg.append("") - paths_with_newlines.sort() - for f in paths_with_newlines: - msg.append("\t/%s" % (f.replace("\n", "\\n").replace("\r", "\\r"))) - msg.append("") - msg.append(_("package %s NOT merged") % self.mycpv) - msg.append("") - eerror(msg) - return 1 - - # If there are no files to merge, and an installed package in the same - # slot has files, it probably means that something went wrong. - if self.settings.get("PORTAGE_PACKAGE_EMPTY_ABORT") == "1" and \ - not filelist and not linklist and others_in_slot: - installed_files = None - for other_dblink in others_in_slot: - installed_files = other_dblink.getcontents() - if not installed_files: - continue - from textwrap import wrap - wrap_width = 72 - msg = [] - d = { - "new_cpv":self.mycpv, - "old_cpv":other_dblink.mycpv - } - msg.extend(wrap(_("The '%(new_cpv)s' package will not install " - "any files, but the currently installed '%(old_cpv)s'" - " package has the following files: ") % d, wrap_width)) - msg.append("") - msg.extend(sorted(installed_files)) - msg.append("") - msg.append(_("package %s NOT merged") % self.mycpv) - msg.append("") - msg.extend(wrap( - _("Manually run `emerge --unmerge =%s` if you " - "really want to remove the above files. Set " - "PORTAGE_PACKAGE_EMPTY_ABORT=\"0\" in " - "/etc/portage/make.conf if you do not want to " - "abort in cases like this.") % other_dblink.mycpv, - wrap_width)) - eerror(msg) - if installed_files: - return 1 - - # Make sure the ebuild environment is initialized and that ${T}/elog - # exists for logging of collision-protect eerror messages. - if myebuild is None: - myebuild = os.path.join(inforoot, self.pkg + ".ebuild") - doebuild_environment(myebuild, "preinst", - settings=self.settings, db=mydbapi) - self.settings["REPLACING_VERSIONS"] = " ".join( - [portage.versions.cpv_getversion(other.mycpv) - for other in others_in_slot]) - prepare_build_dirs(settings=self.settings, cleanup=cleanup) - - # check for package collisions - blockers = [] - for blocker in self._blockers or []: - blocker = self.vartree.dbapi._dblink(blocker.cpv) - # It may have been unmerged before lock(s) - # were aquired. - if blocker.exists(): - blockers.append(blocker) - - collisions, dirs_ro, symlink_collisions, plib_collisions = \ - self._collision_protect(srcroot, destroot, - others_in_slot + blockers, filelist, linklist) - - # Check for read-only filesystems. - ro_checker = get_ro_checker() - rofilesystems = ro_checker(dirs_ro) - - if rofilesystems: - msg = _("One or more files installed to this package are " - "set to be installed to read-only filesystems. " - "Please mount the following filesystems as read-write " - "and retry.") - msg = textwrap.wrap(msg, 70) - msg.append("") - for f in rofilesystems: - msg.append("\t%s" % f) - msg.append("") - self._elog("eerror", "preinst", msg) - - msg = _("Package '%s' NOT merged due to read-only file systems.") % \ - self.settings.mycpv - msg += _(" If necessary, refer to your elog " - "messages for the whole content of the above message.") - msg = textwrap.wrap(msg, 70) - eerror(msg) - return 1 - - if symlink_collisions: - # Symlink collisions need to be distinguished from other types - # of collisions, in order to avoid confusion (see bug #409359). - msg = _("Package '%s' has one or more collisions " - "between symlinks and directories, which is explicitly " - "forbidden by PMS section 13.4 (see bug #326685):") % \ - (self.settings.mycpv,) - msg = textwrap.wrap(msg, 70) - msg.append("") - for f in symlink_collisions: - msg.append("\t%s" % os.path.join(destroot, - f.lstrip(os.path.sep))) - msg.append("") - self._elog("eerror", "preinst", msg) - - if collisions: - collision_protect = "collision-protect" in self.settings.features - protect_owned = "protect-owned" in self.settings.features - msg = _("This package will overwrite one or more files that" - " may belong to other packages (see list below).") - if not (collision_protect or protect_owned): - msg += _(" Add either \"collision-protect\" or" - " \"protect-owned\" to FEATURES in" - " make.conf if you would like the merge to abort" - " in cases like this. See the make.conf man page for" - " more information about these features.") - if self.settings.get("PORTAGE_QUIET") != "1": - msg += _(" You can use a command such as" - " `portageq owners / ` to identify the" - " installed package that owns a file. If portageq" - " reports that only one package owns a file then do NOT" - " file a bug report. A bug report is only useful if it" - " identifies at least two or more packages that are known" - " to install the same file(s)." - " If a collision occurs and you" - " can not explain where the file came from then you" - " should simply ignore the collision since there is not" - " enough information to determine if a real problem" - " exists. Please do NOT file a bug report at" - " https://bugs.gentoo.org/ unless you report exactly which" - " two packages install the same file(s). See" - " https://wiki.gentoo.org/wiki/Knowledge_Base:Blockers" - " for tips on how to solve the problem. And once again," - " please do NOT file a bug report unless you have" - " completely understood the above message.") - - self.settings["EBUILD_PHASE"] = "preinst" - from textwrap import wrap - msg = wrap(msg, 70) - if collision_protect: - msg.append("") - msg.append(_("package %s NOT merged") % self.settings.mycpv) - msg.append("") - msg.append(_("Detected file collision(s):")) - msg.append("") - - for f in collisions: - msg.append("\t%s" % \ - os.path.join(destroot, f.lstrip(os.path.sep))) - - eerror(msg) - - owners = None - if collision_protect or protect_owned or symlink_collisions: - msg = [] - msg.append("") - msg.append(_("Searching all installed" - " packages for file collisions...")) - msg.append("") - msg.append(_("Press Ctrl-C to Stop")) - msg.append("") - eerror(msg) - - if len(collisions) > 20: - # get_owners is slow for large numbers of files, so - # don't look them all up. - collisions = collisions[:20] - - pkg_info_strs = {} - self.lockdb() - try: - owners = self.vartree.dbapi._owners.get_owners(collisions) - self.vartree.dbapi.flush_cache() - - for pkg in owners: - pkg = self.vartree.dbapi._pkg_str(pkg.mycpv, None) - pkg_info_str = "%s%s%s" % (pkg, - _slot_separator, pkg.slot) - if pkg.repo != _unknown_repo: - pkg_info_str += "%s%s" % (_repo_separator, - pkg.repo) - pkg_info_strs[pkg] = pkg_info_str - - finally: - self.unlockdb() - - for pkg, owned_files in owners.items(): - msg = [] - msg.append(pkg_info_strs[pkg.mycpv]) - for f in sorted(owned_files): - msg.append("\t%s" % os.path.join(destroot, - f.lstrip(os.path.sep))) - msg.append("") - eerror(msg) - - if not owners: - eerror([_("None of the installed" - " packages claim the file(s)."), ""]) - - symlink_abort_msg =_("Package '%s' NOT merged since it has " - "one or more collisions between symlinks and directories, " - "which is explicitly forbidden by PMS section 13.4 " - "(see bug #326685).") - - # The explanation about the collision and how to solve - # it may not be visible via a scrollback buffer, especially - # if the number of file collisions is large. Therefore, - # show a summary at the end. - abort = False - if symlink_collisions: - abort = True - msg = symlink_abort_msg % (self.settings.mycpv,) - elif collision_protect: - abort = True - msg = _("Package '%s' NOT merged due to file collisions.") % \ - self.settings.mycpv - elif protect_owned and owners: - abort = True - msg = _("Package '%s' NOT merged due to file collisions.") % \ - self.settings.mycpv - else: - msg = _("Package '%s' merged despite file collisions.") % \ - self.settings.mycpv - msg += _(" If necessary, refer to your elog " - "messages for the whole content of the above message.") - eerror(wrap(msg, 70)) - - if abort: - return 1 - - # The merge process may move files out of the image directory, - # which causes invalidation of the .installed flag. - try: - os.unlink(os.path.join( - os.path.dirname(normalize_path(srcroot)), ".installed")) - except OSError as e: - if e.errno != errno.ENOENT: - raise - del e - - self.dbdir = self.dbtmpdir - self.delete() - ensure_dirs(self.dbtmpdir) - - downgrade = False - if self._installed_instance is not None and \ - vercmp(self.mycpv.version, - self._installed_instance.mycpv.version) < 0: - downgrade = True - - if self._installed_instance is not None: - rval = self._pre_merge_backup(self._installed_instance, downgrade) - if rval != os.EX_OK: - showMessage(_("!!! FAILED preinst: ") + - "quickpkg: %s\n" % rval, - level=logging.ERROR, noiselevel=-1) - return rval - - # run preinst script - showMessage(_(">>> Merging %(cpv)s to %(destroot)s\n") % \ - {"cpv":self.mycpv, "destroot":destroot}) - phase = EbuildPhase(background=False, phase="preinst", - scheduler=self._scheduler, settings=self.settings) - phase.start() - a = phase.wait() - - # XXX: Decide how to handle failures here. - if a != os.EX_OK: - showMessage(_("!!! FAILED preinst: ")+str(a)+"\n", - level=logging.ERROR, noiselevel=-1) - return a - - # copy "info" files (like SLOT, CFLAGS, etc.) into the database - for x in os.listdir(inforoot): - self.copyfile(inforoot+"/"+x) - - # write local package counter for recording - if counter is None: - counter = self.vartree.dbapi.counter_tick(mycpv=self.mycpv) - with io.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'), - encoding=_encodings['fs'], errors='strict'), - mode='w', encoding=_encodings['repo.content'], - errors='backslashreplace') as f: - f.write("%s" % counter) - - self.updateprotect() - - #if we have a file containing previously-merged config file md5sums, grab it. - self.vartree.dbapi._fs_lock() - try: - # This prunes any libraries from the registry that no longer - # exist on disk, in case they have been manually removed. - # This has to be done prior to merge, since after merge it - # is non-trivial to distinguish these files from files - # that have just been merged. - plib_registry = self.vartree.dbapi._plib_registry - if plib_registry: - plib_registry.lock() - try: - plib_registry.load() - plib_registry.store() - finally: - plib_registry.unlock() - - # Always behave like --noconfmem is enabled for downgrades - # so that people who don't know about this option are less - # likely to get confused when doing upgrade/downgrade cycles. - cfgfiledict = grabdict(self.vartree.dbapi._conf_mem_file) - if "NOCONFMEM" in self.settings or downgrade: - cfgfiledict["IGNORE"]=1 - else: - cfgfiledict["IGNORE"]=0 - - rval = self._merge_contents(srcroot, destroot, cfgfiledict) - if rval != os.EX_OK: - return rval - finally: - self.vartree.dbapi._fs_unlock() - - # These caches are populated during collision-protect and the data - # they contain is now invalid. It's very important to invalidate - # the contents_inodes cache so that FEATURES=unmerge-orphans - # doesn't unmerge anything that belongs to this package that has - # just been merged. - for dblnk in others_in_slot: - dblnk._clear_contents_cache() - self._clear_contents_cache() - - linkmap = self.vartree.dbapi._linkmap - plib_registry = self.vartree.dbapi._plib_registry - # We initialize preserve_paths to an empty set rather - # than None here because it plays an important role - # in prune_plib_registry logic by serving to indicate - # that we have a replacement for a package that's - # being unmerged. - - preserve_paths = set() - needed = None - if not (self._linkmap_broken or linkmap is None or - plib_registry is None): - self.vartree.dbapi._fs_lock() - plib_registry.lock() - try: - plib_registry.load() - needed = os.path.join(inforoot, linkmap._needed_aux_key) - self._linkmap_rebuild(include_file=needed) - - # Preserve old libs if they are still in use - # TODO: Handle cases where the previous instance - # has already been uninstalled but it still has some - # preserved libraries in the registry that we may - # want to preserve here. - preserve_paths = self._find_libs_to_preserve() - finally: - plib_registry.unlock() - self.vartree.dbapi._fs_unlock() - - if preserve_paths: - self._add_preserve_libs_to_contents(preserve_paths) - - # If portage is reinstalling itself, remove the old - # version now since we want to use the temporary - # PORTAGE_BIN_PATH that will be removed when we return. - reinstall_self = False - if self.myroot == "/" and \ - match_from_list(PORTAGE_PACKAGE_ATOM, [self.mycpv]): - reinstall_self = True - - emerge_log = self._emerge_log - - # If we have any preserved libraries then autoclean - # is forced so that preserve-libs logic doesn't have - # to account for the additional complexity of the - # AUTOCLEAN=no mode. - autoclean = self.settings.get("AUTOCLEAN", "yes") == "yes" \ - or preserve_paths - - if autoclean: - emerge_log(_(" >>> AUTOCLEAN: %s") % (slot_atom,)) - - others_in_slot.append(self) # self has just been merged - for dblnk in list(others_in_slot): - if dblnk is self: - continue - if not (autoclean or dblnk.mycpv == self.mycpv or reinstall_self): - continue - showMessage(_(">>> Safely unmerging already-installed instance...\n")) - emerge_log(_(" === Unmerging... (%s)") % (dblnk.mycpv,)) - others_in_slot.remove(dblnk) # dblnk will unmerge itself now - dblnk._linkmap_broken = self._linkmap_broken - dblnk.settings["REPLACED_BY_VERSION"] = portage.versions.cpv_getversion(self.mycpv) - dblnk.settings.backup_changes("REPLACED_BY_VERSION") - unmerge_rval = dblnk.unmerge(ldpath_mtimes=prev_mtimes, - others_in_slot=others_in_slot, needed=needed, - preserve_paths=preserve_paths) - dblnk.settings.pop("REPLACED_BY_VERSION", None) - - if unmerge_rval == os.EX_OK: - emerge_log(_(" >>> unmerge success: %s") % (dblnk.mycpv,)) - else: - emerge_log(_(" !!! unmerge FAILURE: %s") % (dblnk.mycpv,)) - - self.lockdb() - try: - # TODO: Check status and abort if necessary. - dblnk.delete() - finally: - self.unlockdb() - showMessage(_(">>> Original instance of package unmerged safely.\n")) - - if len(others_in_slot) > 1: - showMessage(colorize("WARN", _("WARNING:")) - + _(" AUTOCLEAN is disabled. This can cause serious" - " problems due to overlapping packages.\n"), - level=logging.WARN, noiselevel=-1) - - # We hold both directory locks. - self.dbdir = self.dbpkgdir - self.lockdb() - try: - self.delete() - _movefile(self.dbtmpdir, self.dbpkgdir, mysettings=self.settings) - self._merged_path(self.dbpkgdir, os.lstat(self.dbpkgdir)) - self.vartree.dbapi._cache_delta.recordEvent( - "add", self.mycpv, slot, counter) - finally: - self.unlockdb() - - # Check for file collisions with blocking packages - # and remove any colliding files from their CONTENTS - # since they now belong to this package. - self._clear_contents_cache() - contents = self.getcontents() - destroot_len = len(destroot) - 1 - self.lockdb() - try: - for blocker in blockers: - self.vartree.dbapi.removeFromContents(blocker, iter(contents), - relative_paths=False) - finally: - self.unlockdb() - - plib_registry = self.vartree.dbapi._plib_registry - if plib_registry: - self.vartree.dbapi._fs_lock() - plib_registry.lock() - try: - plib_registry.load() - - if preserve_paths: - # keep track of the libs we preserved - plib_registry.register(self.mycpv, slot, counter, - sorted(preserve_paths)) - - # Unregister any preserved libs that this package has overwritten - # and update the contents of the packages that owned them. - plib_dict = plib_registry.getPreservedLibs() - for cpv, paths in plib_collisions.items(): - if cpv not in plib_dict: - continue - has_vdb_entry = False - if cpv != self.mycpv: - # If we've replaced another instance with the - # same cpv then the vdb entry no longer belongs - # to it, so we'll have to get the slot and counter - # from plib_registry._data instead. - self.vartree.dbapi.lock() - try: - try: - slot = self.vartree.dbapi._pkg_str(cpv, None).slot - counter = self.vartree.dbapi.cpv_counter(cpv) - except (KeyError, InvalidData): - pass - else: - has_vdb_entry = True - self.vartree.dbapi.removeFromContents( - cpv, paths) - finally: - self.vartree.dbapi.unlock() - - if not has_vdb_entry: - # It's possible for previously unmerged packages - # to have preserved libs in the registry, so try - # to retrieve the slot and counter from there. - has_registry_entry = False - for plib_cps, (plib_cpv, plib_counter, plib_paths) in \ - plib_registry._data.items(): - if plib_cpv != cpv: - continue - try: - cp, slot = plib_cps.split(":", 1) - except ValueError: - continue - counter = plib_counter - has_registry_entry = True - break - - if not has_registry_entry: - continue - - remaining = [f for f in plib_dict[cpv] if f not in paths] - plib_registry.register(cpv, slot, counter, remaining) - - plib_registry.store() - finally: - plib_registry.unlock() - self.vartree.dbapi._fs_unlock() - - self.vartree.dbapi._add(self) - contents = self.getcontents() - - #do postinst script - self.settings["PORTAGE_UPDATE_ENV"] = \ - os.path.join(self.dbpkgdir, "environment.bz2") - self.settings.backup_changes("PORTAGE_UPDATE_ENV") - try: - phase = EbuildPhase(background=False, phase="postinst", - scheduler=self._scheduler, settings=self.settings) - phase.start() - a = phase.wait() - if a == os.EX_OK: - showMessage(_(">>> %s merged.\n") % self.mycpv) - finally: - self.settings.pop("PORTAGE_UPDATE_ENV", None) - - if a != os.EX_OK: - # It's stupid to bail out here, so keep going regardless of - # phase return code. - self._postinst_failure = True - self._elog("eerror", "postinst", [ - _("FAILED postinst: %s") % (a,), - ]) - - #update environment settings, library paths. DO NOT change symlinks. - env_update( - target_root=self.settings['ROOT'], prev_mtimes=prev_mtimes, - contents=contents, env=self.settings, - writemsg_level=self._display_merge, vardbapi=self.vartree.dbapi) - - # For gcc upgrades, preserved libs have to be removed after the - # the library path has been updated. - self._prune_plib_registry() - self._post_merge_sync() - - return os.EX_OK - - def _new_backup_path(self, p): - """ - The works for any type path, such as a regular file, symlink, - or directory. The parent directory is assumed to exist. - The returned filename is of the form p + '.backup.' + x, where - x guarantees that the returned path does not exist yet. - """ - os = _os_merge - - x = -1 - while True: - x += 1 - backup_p = '%s.backup.%04d' % (p, x) - try: - os.lstat(backup_p) - except OSError: - break - - return backup_p - - def _merge_contents(self, srcroot, destroot, cfgfiledict): - - cfgfiledict_orig = cfgfiledict.copy() - - # open CONTENTS file (possibly overwriting old one) for recording - # Use atomic_ofstream for automatic coercion of raw bytes to - # unicode, in order to prevent TypeError when writing raw bytes - # to TextIOWrapper with python2. - outfile = atomic_ofstream(_unicode_encode( - os.path.join(self.dbtmpdir, 'CONTENTS'), - encoding=_encodings['fs'], errors='strict'), - mode='w', encoding=_encodings['repo.content'], - errors='backslashreplace') - - # Don't bump mtimes on merge since some application require - # preservation of timestamps. This means that the unmerge phase must - # check to see if file belongs to an installed instance in the same - # slot. - mymtime = None - - # set umask to 0 for merging; back up umask, save old one in prevmask (since this is a global change) - prevmask = os.umask(0) - secondhand = [] - - # we do a first merge; this will recurse through all files in our srcroot but also build up a - # "second hand" of symlinks to merge later - if self.mergeme(srcroot, destroot, outfile, secondhand, - self.settings["EPREFIX"].lstrip(os.sep), cfgfiledict, mymtime): - return 1 - - # now, it's time for dealing our second hand; we'll loop until we can't merge anymore. The rest are - # broken symlinks. We'll merge them too. - lastlen = 0 - while len(secondhand) and len(secondhand)!=lastlen: - # clear the thirdhand. Anything from our second hand that - # couldn't get merged will be added to thirdhand. - - thirdhand = [] - if self.mergeme(srcroot, destroot, outfile, thirdhand, - secondhand, cfgfiledict, mymtime): - return 1 - - #swap hands - lastlen = len(secondhand) - - # our thirdhand now becomes our secondhand. It's ok to throw - # away secondhand since thirdhand contains all the stuff that - # couldn't be merged. - secondhand = thirdhand - - if len(secondhand): - # force merge of remaining symlinks (broken or circular; oh well) - if self.mergeme(srcroot, destroot, outfile, None, - secondhand, cfgfiledict, mymtime): - return 1 - - #restore umask - os.umask(prevmask) - - #if we opened it, close it - outfile.flush() - outfile.close() - - # write out our collection of md5sums - if cfgfiledict != cfgfiledict_orig: - cfgfiledict.pop("IGNORE", None) - try: - writedict(cfgfiledict, self.vartree.dbapi._conf_mem_file) - except InvalidLocation: - self.settings._init_dirs() - writedict(cfgfiledict, self.vartree.dbapi._conf_mem_file) - - return os.EX_OK - - def mergeme(self, srcroot, destroot, outfile, secondhand, stufftomerge, cfgfiledict, thismtime): - """ - - This function handles actual merging of the package contents to the livefs. - It also handles config protection. - - @param srcroot: Where are we copying files from (usually ${D}) - @type srcroot: String (Path) - @param destroot: Typically ${ROOT} - @type destroot: String (Path) - @param outfile: File to log operations to - @type outfile: File Object - @param secondhand: A set of items to merge in pass two (usually - or symlinks that point to non-existing files that may get merged later) - @type secondhand: List - @param stufftomerge: Either a diretory to merge, or a list of items. - @type stufftomerge: String or List - @param cfgfiledict: { File:mtime } mapping for config_protected files - @type cfgfiledict: Dictionary - @param thismtime: None or new mtime for merged files (expressed in seconds - in Python <3.3 and nanoseconds in Python >=3.3) - @type thismtime: None or Int - @rtype: None or Boolean - @return: - 1. True on failure - 2. None otherwise - - """ - - showMessage = self._display_merge - writemsg = self._display_merge - - os = _os_merge - sep = os.sep - join = os.path.join - srcroot = normalize_path(srcroot).rstrip(sep) + sep - destroot = normalize_path(destroot).rstrip(sep) + sep - calc_prelink = "prelink-checksums" in self.settings.features - - protect_if_modified = \ - "config-protect-if-modified" in self.settings.features and \ - self._installed_instance is not None - - # this is supposed to merge a list of files. There will be 2 forms of argument passing. - if isinstance(stufftomerge, basestring): - #A directory is specified. Figure out protection paths, listdir() it and process it. - mergelist = [join(stufftomerge, child) for child in \ - os.listdir(join(srcroot, stufftomerge))] - else: - mergelist = stufftomerge[:] - - while mergelist: - - relative_path = mergelist.pop() - mysrc = join(srcroot, relative_path) - mydest = join(destroot, relative_path) - # myrealdest is mydest without the $ROOT prefix (makes a difference if ROOT!="/") - myrealdest = join(sep, relative_path) - # stat file once, test using S_* macros many times (faster that way) - mystat = os.lstat(mysrc) - mymode = mystat[stat.ST_MODE] - mymd5 = None - myto = None - - if sys.hexversion >= 0x3030000: - mymtime = mystat.st_mtime_ns - else: - mymtime = mystat[stat.ST_MTIME] - - if stat.S_ISREG(mymode): - mymd5 = perform_md5(mysrc, calc_prelink=calc_prelink) - elif stat.S_ISLNK(mymode): - # The file name of mysrc and the actual file that it points to - # will have earlier been forcefully converted to the 'merge' - # encoding if necessary, but the content of the symbolic link - # may need to be forcefully converted here. - myto = _os.readlink(_unicode_encode(mysrc, - encoding=_encodings['merge'], errors='strict')) - try: - myto = _unicode_decode(myto, - encoding=_encodings['merge'], errors='strict') - except UnicodeDecodeError: - myto = _unicode_decode(myto, encoding=_encodings['merge'], - errors='replace') - myto = _unicode_encode(myto, encoding='ascii', - errors='backslashreplace') - myto = _unicode_decode(myto, encoding=_encodings['merge'], - errors='replace') - os.unlink(mysrc) - os.symlink(myto, mysrc) - - mymd5 = md5(_unicode_encode(myto)).hexdigest() - - protected = False - if stat.S_ISLNK(mymode) or stat.S_ISREG(mymode): - protected = self.isprotected(mydest) - - if stat.S_ISREG(mymode) and \ - mystat.st_size == 0 and \ - os.path.basename(mydest).startswith(".keep"): - protected = False - - destmd5 = None - mydest_link = None - # handy variables; mydest is the target object on the live filesystems; - # mysrc is the source object in the temporary install dir - try: - mydstat = os.lstat(mydest) - mydmode = mydstat.st_mode - if protected: - if stat.S_ISLNK(mydmode): - # Read symlink target as bytes, in case the - # target path has a bad encoding. - mydest_link = _os.readlink( - _unicode_encode(mydest, - encoding=_encodings['merge'], - errors='strict')) - mydest_link = _unicode_decode(mydest_link, - encoding=_encodings['merge'], - errors='replace') - - # For protection of symlinks, the md5 - # of the link target path string is used - # for cfgfiledict (symlinks are - # protected since bug #485598). - destmd5 = md5(_unicode_encode(mydest_link)).hexdigest() - - elif stat.S_ISREG(mydmode): - destmd5 = perform_md5(mydest, - calc_prelink=calc_prelink) - except (FileNotFound, OSError) as e: - if isinstance(e, OSError) and e.errno != errno.ENOENT: - raise - #dest file doesn't exist - mydstat = None - mydmode = None - mydest_link = None - destmd5 = None - - moveme = True - if protected: - mydest, protected, moveme = self._protect(cfgfiledict, - protect_if_modified, mymd5, myto, mydest, - myrealdest, mydmode, destmd5, mydest_link) - - zing = "!!!" - if not moveme: - # confmem rejected this update - zing = "---" - - if stat.S_ISLNK(mymode): - # we are merging a symbolic link - # Pass in the symlink target in order to bypass the - # os.readlink() call inside abssymlink(), since that - # call is unsafe if the merge encoding is not ascii - # or utf_8 (see bug #382021). - myabsto = abssymlink(mysrc, target=myto) - - if myabsto.startswith(srcroot): - myabsto = myabsto[len(srcroot):] - myabsto = myabsto.lstrip(sep) - if self.settings and self.settings["D"]: - if myto.startswith(self.settings["D"]): - myto = myto[len(self.settings["D"])-1:] - # myrealto contains the path of the real file to which this symlink points. - # we can simply test for existence of this file to see if the target has been merged yet - myrealto = normalize_path(os.path.join(destroot, myabsto)) - if mydmode is not None and stat.S_ISDIR(mydmode): - if not protected: - # we can't merge a symlink over a directory - newdest = self._new_backup_path(mydest) - msg = [] - msg.append("") - msg.append(_("Installation of a symlink is blocked by a directory:")) - msg.append(" '%s'" % mydest) - msg.append(_("This symlink will be merged with a different name:")) - msg.append(" '%s'" % newdest) - msg.append("") - self._eerror("preinst", msg) - mydest = newdest - - # if secondhand is None it means we're operating in "force" mode and should not create a second hand. - if (secondhand != None) and (not os.path.exists(myrealto)): - # either the target directory doesn't exist yet or the target file doesn't exist -- or - # the target is a broken symlink. We will add this file to our "second hand" and merge - # it later. - secondhand.append(mysrc[len(srcroot):]) - continue - # unlinking no longer necessary; "movefile" will overwrite symlinks atomically and correctly - if moveme: - zing = ">>>" - mymtime = movefile(mysrc, mydest, newmtime=thismtime, - sstat=mystat, mysettings=self.settings, - encoding=_encodings['merge']) - - try: - self._merged_path(mydest, os.lstat(mydest)) - except OSError: - pass - - if mymtime != None: - # Use lexists, since if the target happens to be a broken - # symlink then that should trigger an independent warning. - if not (os.path.lexists(myrealto) or - os.path.lexists(join(srcroot, myabsto))): - self._eqawarn('preinst', - [_("QA Notice: Symbolic link /%s points to /%s which does not exist.") - % (relative_path, myabsto)]) - - showMessage("%s %s -> %s\n" % (zing, mydest, myto)) - if sys.hexversion >= 0x3030000: - outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime // 1000000000)+"\n") - else: - outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime)+"\n") - else: - showMessage(_("!!! Failed to move file.\n"), - level=logging.ERROR, noiselevel=-1) - showMessage("!!! %s -> %s\n" % (mydest, myto), - level=logging.ERROR, noiselevel=-1) - return 1 - elif stat.S_ISDIR(mymode): - # we are merging a directory - if mydmode != None: - # destination exists - - if bsd_chflags: - # Save then clear flags on dest. - dflags = mydstat.st_flags - if dflags != 0: - bsd_chflags.lchflags(mydest, 0) - - if not stat.S_ISLNK(mydmode) and \ - not os.access(mydest, os.W_OK): - pkgstuff = pkgsplit(self.pkg) - writemsg(_("\n!!! Cannot write to '%s'.\n") % mydest, noiselevel=-1) - writemsg(_("!!! Please check permissions and directories for broken symlinks.\n")) - writemsg(_("!!! You may start the merge process again by using ebuild:\n")) - writemsg("!!! ebuild "+self.settings["PORTDIR"]+"/"+self.cat+"/"+pkgstuff[0]+"/"+self.pkg+".ebuild merge\n") - writemsg(_("!!! And finish by running this: env-update\n\n")) - return 1 - - if stat.S_ISDIR(mydmode) or \ - (stat.S_ISLNK(mydmode) and os.path.isdir(mydest)): - # a symlink to an existing directory will work for us; keep it: - showMessage("--- %s/\n" % mydest) - if bsd_chflags: - bsd_chflags.lchflags(mydest, dflags) - else: - # a non-directory and non-symlink-to-directory. Won't work for us. Move out of the way. - backup_dest = self._new_backup_path(mydest) - msg = [] - msg.append("") - msg.append(_("Installation of a directory is blocked by a file:")) - msg.append(" '%s'" % mydest) - msg.append(_("This file will be renamed to a different name:")) - msg.append(" '%s'" % backup_dest) - msg.append("") - self._eerror("preinst", msg) - if movefile(mydest, backup_dest, - mysettings=self.settings, - encoding=_encodings['merge']) is None: - return 1 - showMessage(_("bak %s %s.backup\n") % (mydest, mydest), - level=logging.ERROR, noiselevel=-1) - #now create our directory - try: - if self.settings.selinux_enabled(): - _selinux_merge.mkdir(mydest, mysrc) - else: - os.mkdir(mydest) - except OSError as e: - # Error handling should be equivalent to - # portage.util.ensure_dirs() for cases - # like bug #187518. - if e.errno in (errno.EEXIST,): - pass - elif os.path.isdir(mydest): - pass - else: - raise - del e - - if bsd_chflags: - bsd_chflags.lchflags(mydest, dflags) - os.chmod(mydest, mystat[0]) - os.chown(mydest, mystat[4], mystat[5]) - showMessage(">>> %s/\n" % mydest) - else: - try: - #destination doesn't exist - if self.settings.selinux_enabled(): - _selinux_merge.mkdir(mydest, mysrc) - else: - os.mkdir(mydest) - except OSError as e: - # Error handling should be equivalent to - # portage.util.ensure_dirs() for cases - # like bug #187518. - if e.errno in (errno.EEXIST,): - pass - elif os.path.isdir(mydest): - pass - else: - raise - del e - os.chmod(mydest, mystat[0]) - os.chown(mydest, mystat[4], mystat[5]) - showMessage(">>> %s/\n" % mydest) - - try: - self._merged_path(mydest, os.lstat(mydest)) - except OSError: - pass - - outfile.write("dir "+myrealdest+"\n") - # recurse and merge this directory - mergelist.extend(join(relative_path, child) for child in - os.listdir(join(srcroot, relative_path))) - - elif stat.S_ISREG(mymode): - # we are merging a regular file - if not protected and \ - mydmode is not None and stat.S_ISDIR(mydmode): - # install of destination is blocked by an existing directory with the same name - newdest = self._new_backup_path(mydest) - msg = [] - msg.append("") - msg.append(_("Installation of a regular file is blocked by a directory:")) - msg.append(" '%s'" % mydest) - msg.append(_("This file will be merged with a different name:")) - msg.append(" '%s'" % newdest) - msg.append("") - self._eerror("preinst", msg) - mydest = newdest - - # whether config protection or not, we merge the new file the - # same way. Unless moveme=0 (blocking directory) - if moveme: - # Create hardlinks only for source files that already exist - # as hardlinks (having identical st_dev and st_ino). - hardlink_key = (mystat.st_dev, mystat.st_ino) - - hardlink_candidates = self._hardlink_merge_map.get(hardlink_key) - if hardlink_candidates is None: - hardlink_candidates = [] - self._hardlink_merge_map[hardlink_key] = hardlink_candidates - - mymtime = movefile(mysrc, mydest, newmtime=thismtime, - sstat=mystat, mysettings=self.settings, - hardlink_candidates=hardlink_candidates, - encoding=_encodings['merge']) - if mymtime is None: - return 1 - hardlink_candidates.append(mydest) - zing = ">>>" - - try: - self._merged_path(mydest, os.lstat(mydest)) - except OSError: - pass - - if mymtime != None: - if sys.hexversion >= 0x3030000: - outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime // 1000000000)+"\n") - else: - outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime)+"\n") - showMessage("%s %s\n" % (zing,mydest)) - else: - # we are merging a fifo or device node - zing = "!!!" - if mydmode is None: - # destination doesn't exist - if movefile(mysrc, mydest, newmtime=thismtime, - sstat=mystat, mysettings=self.settings, - encoding=_encodings['merge']) is not None: - zing = ">>>" - - try: - self._merged_path(mydest, os.lstat(mydest)) - except OSError: - pass - - else: - return 1 - if stat.S_ISFIFO(mymode): - outfile.write("fif %s\n" % myrealdest) - else: - outfile.write("dev %s\n" % myrealdest) - showMessage(zing + " " + mydest + "\n") - - def _protect(self, cfgfiledict, protect_if_modified, src_md5, - src_link, dest, dest_real, dest_mode, dest_md5, dest_link): - - move_me = True - protected = True - force = False - k = False - if self._installed_instance is not None: - k = self._installed_instance._match_contents(dest_real) - if k is not False: - if dest_mode is None: - # If the file doesn't exist, then it may - # have been deleted or renamed by the - # admin. Therefore, force the file to be - # merged with a ._cfg name, so that the - # admin will be prompted for this update - # (see bug #523684). - force = True - - elif protect_if_modified: - data = self._installed_instance.getcontents()[k] - if data[0] == "obj" and data[2] == dest_md5: - protected = False - elif data[0] == "sym" and data[2] == dest_link: - protected = False - - if protected and dest_mode is not None: - # we have a protection path; enable config file management. - if src_md5 == dest_md5: - protected = False - - elif src_md5 == cfgfiledict.get(dest_real, [None])[0]: - # An identical update has previously been - # merged. Skip it unless the user has chosen - # --noconfmem. - move_me = protected = bool(cfgfiledict["IGNORE"]) - - if protected and \ - (dest_link is not None or src_link is not None) and \ - dest_link != src_link: - # If either one is a symlink, and they are not - # identical symlinks, then force config protection. - force = True - - if move_me: - # Merging a new file, so update confmem. - cfgfiledict[dest_real] = [src_md5] - elif dest_md5 == cfgfiledict.get(dest_real, [None])[0]: - # A previously remembered update has been - # accepted, so it is removed from confmem. - del cfgfiledict[dest_real] - - if protected and move_me: - dest = new_protect_filename(dest, - newmd5=(dest_link or src_md5), - force=force) - - return dest, protected, move_me - - def _merged_path(self, path, lstatobj, exists=True): - previous_path = self._device_path_map.get(lstatobj.st_dev) - if previous_path is None or previous_path is False or \ - (exists and len(path) < len(previous_path)): - if exists: - self._device_path_map[lstatobj.st_dev] = path - else: - # This entry is used to indicate that we've unmerged - # a file from this device, and later, this entry is - # replaced by a parent directory. - self._device_path_map[lstatobj.st_dev] = False - - def _post_merge_sync(self): - """ - Call this after merge or unmerge, in order to sync relevant files to - disk and avoid data-loss in the event of a power failure. This method - does nothing if FEATURES=merge-sync is disabled. - """ - if not self._device_path_map or \ - "merge-sync" not in self.settings.features: - return - - returncode = None - if platform.system() == "Linux": - - paths = [] - for path in self._device_path_map.values(): - if path is not False: - paths.append(path) - paths = tuple(paths) - - proc = SyncfsProcess(paths=paths, - scheduler=(self._scheduler or - portage._internal_caller and global_event_loop() or - EventLoop(main=False))) - proc.start() - returncode = proc.wait() - - if returncode is None or returncode != os.EX_OK: - try: - proc = subprocess.Popen(["sync"]) - except EnvironmentError: - pass - else: - proc.wait() - - @_slot_locked - def merge(self, mergeroot, inforoot, myroot=None, myebuild=None, cleanup=0, - mydbapi=None, prev_mtimes=None, counter=None): - """ - @param myroot: ignored, self._eroot is used instead - """ - myroot = None - retval = -1 - parallel_install = "parallel-install" in self.settings.features - if not parallel_install: - self.lockdb() - self.vartree.dbapi._bump_mtime(self.mycpv) - if self._scheduler is None: - self._scheduler = SchedulerInterface(portage._internal_caller and - global_event_loop() or EventLoop(main=False)) - try: - retval = self.treewalk(mergeroot, myroot, inforoot, myebuild, - cleanup=cleanup, mydbapi=mydbapi, prev_mtimes=prev_mtimes, - counter=counter) - - # If PORTAGE_BUILDDIR doesn't exist, then it probably means - # fail-clean is enabled, and the success/die hooks have - # already been called by EbuildPhase. - if os.path.isdir(self.settings['PORTAGE_BUILDDIR']): - - if retval == os.EX_OK: - phase = 'success_hooks' - else: - phase = 'die_hooks' - - ebuild_phase = MiscFunctionsProcess( - background=False, commands=[phase], - scheduler=self._scheduler, settings=self.settings) - ebuild_phase.start() - ebuild_phase.wait() - self._elog_process() - - if 'noclean' not in self.settings.features and \ - (retval == os.EX_OK or \ - 'fail-clean' in self.settings.features): - if myebuild is None: - myebuild = os.path.join(inforoot, self.pkg + ".ebuild") - - doebuild_environment(myebuild, "clean", - settings=self.settings, db=mydbapi) - phase = EbuildPhase(background=False, phase="clean", - scheduler=self._scheduler, settings=self.settings) - phase.start() - phase.wait() - finally: - self.settings.pop('REPLACING_VERSIONS', None) - if self.vartree.dbapi._linkmap is None: - # preserve-libs is entirely disabled - pass - else: - self.vartree.dbapi._linkmap._clear_cache() - self.vartree.dbapi._bump_mtime(self.mycpv) - if not parallel_install: - self.unlockdb() - - if retval == os.EX_OK and self._postinst_failure: - retval = portage.const.RETURNCODE_POSTINST_FAILURE - - return retval - - def getstring(self,name): - "returns contents of a file with whitespace converted to spaces" - if not os.path.exists(self.dbdir+"/"+name): - return "" - with io.open( - _unicode_encode(os.path.join(self.dbdir, name), - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], errors='replace' - ) as f: - mydata = f.read().split() - return " ".join(mydata) - - def copyfile(self,fname): - shutil.copyfile(fname,self.dbdir+"/"+os.path.basename(fname)) - - def getfile(self,fname): - if not os.path.exists(self.dbdir+"/"+fname): - return "" - with io.open(_unicode_encode(os.path.join(self.dbdir, fname), - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], errors='replace' - ) as f: - return f.read() - - def setfile(self,fname,data): - kwargs = {} - if fname == 'environment.bz2' or not isinstance(data, basestring): - kwargs['mode'] = 'wb' - else: - kwargs['mode'] = 'w' - kwargs['encoding'] = _encodings['repo.content'] - write_atomic(os.path.join(self.dbdir, fname), data, **kwargs) - - def getelements(self,ename): - if not os.path.exists(self.dbdir+"/"+ename): - return [] - with io.open(_unicode_encode( - os.path.join(self.dbdir, ename), - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], errors='replace' - ) as f: - mylines = f.readlines() - myreturn = [] - for x in mylines: - for y in x[:-1].split(): - myreturn.append(y) - return myreturn - - def setelements(self,mylist,ename): - with io.open(_unicode_encode( - os.path.join(self.dbdir, ename), - encoding=_encodings['fs'], errors='strict'), - mode='w', encoding=_encodings['repo.content'], - errors='backslashreplace') as f: - for x in mylist: - f.write("%s\n" % x) - - def isregular(self): - "Is this a regular package (does it have a CATEGORY file? A dblink can be virtual *and* regular)" - return os.path.exists(os.path.join(self.dbdir, "CATEGORY")) - - def _pre_merge_backup(self, backup_dblink, downgrade): - - if ("unmerge-backup" in self.settings.features or - (downgrade and "downgrade-backup" in self.settings.features)): - return self._quickpkg_dblink(backup_dblink, False, None) - - return os.EX_OK - - def _pre_unmerge_backup(self, background): - - if "unmerge-backup" in self.settings.features : - logfile = None - if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": - logfile = self.settings.get("PORTAGE_LOG_FILE") - return self._quickpkg_dblink(self, background, logfile) - - return os.EX_OK - - def _quickpkg_dblink(self, backup_dblink, background, logfile): - - build_time = backup_dblink.getfile('BUILD_TIME') - try: - build_time = long(build_time.strip()) - except ValueError: - build_time = 0 - - trees = QueryCommand.get_db()[self.settings["EROOT"]] - bintree = trees["bintree"] - - for binpkg in reversed( - bintree.dbapi.match('={}'.format(backup_dblink.mycpv))): - if binpkg.build_time == build_time: - return os.EX_OK - - self.lockdb() - try: - - if not backup_dblink.exists(): - # It got unmerged by a concurrent process. - return os.EX_OK - - # Call quickpkg for support of QUICKPKG_DEFAULT_OPTS and stuff. - quickpkg_binary = os.path.join(self.settings["PORTAGE_BIN_PATH"], - "quickpkg") - - if not os.access(quickpkg_binary, os.X_OK): - # If not running from the source tree, use PATH. - quickpkg_binary = find_binary("quickpkg") - if quickpkg_binary is None: - self._display_merge( - _("%s: command not found") % "quickpkg", - level=logging.ERROR, noiselevel=-1) - return 127 - - # Let quickpkg inherit the global vartree config's env. - env = dict(self.vartree.settings.items()) - env["__PORTAGE_INHERIT_VARDB_LOCK"] = "1" - - pythonpath = [x for x in env.get('PYTHONPATH', '').split(":") if x] - if not pythonpath or \ - not os.path.samefile(pythonpath[0], portage._pym_path): - pythonpath.insert(0, portage._pym_path) - env['PYTHONPATH'] = ":".join(pythonpath) - - quickpkg_proc = SpawnProcess( - args=[portage._python_interpreter, quickpkg_binary, - "=%s" % (backup_dblink.mycpv,)], - background=background, env=env, - scheduler=self._scheduler, logfile=logfile) - quickpkg_proc.start() - - return quickpkg_proc.wait() - - finally: - self.unlockdb() - -def merge(mycat, mypkg, pkgloc, infloc, - myroot=None, settings=None, myebuild=None, - mytree=None, mydbapi=None, vartree=None, prev_mtimes=None, blockers=None, - scheduler=None, fd_pipes=None): - """ - @param myroot: ignored, settings['EROOT'] is used instead - """ - myroot = None - if settings is None: - raise TypeError("settings argument is required") - if not os.access(settings['EROOT'], os.W_OK): - writemsg(_("Permission denied: access('%s', W_OK)\n") % settings['EROOT'], - noiselevel=-1) - return errno.EACCES - background = (settings.get('PORTAGE_BACKGROUND') == '1') - merge_task = MergeProcess( - mycat=mycat, mypkg=mypkg, settings=settings, - treetype=mytree, vartree=vartree, - scheduler=(scheduler or portage._internal_caller and - global_event_loop() or EventLoop(main=False)), - background=background, blockers=blockers, pkgloc=pkgloc, - infloc=infloc, myebuild=myebuild, mydbapi=mydbapi, - prev_mtimes=prev_mtimes, logfile=settings.get('PORTAGE_LOG_FILE'), - fd_pipes=fd_pipes) - merge_task.start() - retcode = merge_task.wait() - return retcode - -def unmerge(cat, pkg, myroot=None, settings=None, - mytrimworld=None, vartree=None, - ldpath_mtimes=None, scheduler=None): - """ - @param myroot: ignored, settings['EROOT'] is used instead - @param mytrimworld: ignored - """ - myroot = None - if settings is None: - raise TypeError("settings argument is required") - mylink = dblink(cat, pkg, settings=settings, treetype="vartree", - vartree=vartree, scheduler=scheduler) - vartree = mylink.vartree - parallel_install = "parallel-install" in settings.features - if not parallel_install: - mylink.lockdb() - try: - if mylink.exists(): - retval = mylink.unmerge(ldpath_mtimes=ldpath_mtimes) - if retval == os.EX_OK: - mylink.lockdb() - try: - mylink.delete() - finally: - mylink.unlockdb() - return retval - return os.EX_OK - finally: - if vartree.dbapi._linkmap is None: - # preserve-libs is entirely disabled - pass - else: - vartree.dbapi._linkmap._clear_cache() - if not parallel_install: - mylink.unlockdb() - -def write_contents(contents, root, f): - """ - Write contents to any file like object. The file will be left open. - """ - root_len = len(root) - 1 - for filename in sorted(contents): - entry_data = contents[filename] - entry_type = entry_data[0] - relative_filename = filename[root_len:] - if entry_type == "obj": - entry_type, mtime, md5sum = entry_data - line = "%s %s %s %s\n" % \ - (entry_type, relative_filename, md5sum, mtime) - elif entry_type == "sym": - entry_type, mtime, link = entry_data - line = "%s %s -> %s %s\n" % \ - (entry_type, relative_filename, link, mtime) - else: # dir, dev, fif - line = "%s %s\n" % (entry_type, relative_filename) - f.write(line) - -def tar_contents(contents, root, tar, protect=None, onProgress=None, - xattrs=False): - os = _os_merge - encoding = _encodings['merge'] - - try: - for x in contents: - _unicode_encode(x, - encoding=_encodings['merge'], - errors='strict') - except UnicodeEncodeError: - # The package appears to have been merged with a - # different value of sys.getfilesystemencoding(), - # so fall back to utf_8 if appropriate. - try: - for x in contents: - _unicode_encode(x, - encoding=_encodings['fs'], - errors='strict') - except UnicodeEncodeError: - pass - else: - os = portage.os - encoding = _encodings['fs'] - - tar.encoding = encoding - root = normalize_path(root).rstrip(os.path.sep) + os.path.sep - id_strings = {} - maxval = len(contents) - curval = 0 - if onProgress: - onProgress(maxval, 0) - paths = list(contents) - paths.sort() - for path in paths: - curval += 1 - try: - lst = os.lstat(path) - except OSError as e: - if e.errno != errno.ENOENT: - raise - del e - if onProgress: - onProgress(maxval, curval) - continue - contents_type = contents[path][0] - if path.startswith(root): - arcname = "./" + path[len(root):] - else: - raise ValueError("invalid root argument: '%s'" % root) - live_path = path - if 'dir' == contents_type and \ - not stat.S_ISDIR(lst.st_mode) and \ - os.path.isdir(live_path): - # Even though this was a directory in the original ${D}, it exists - # as a symlink to a directory in the live filesystem. It must be - # recorded as a real directory in the tar file to ensure that tar - # can properly extract it's children. - live_path = os.path.realpath(live_path) - lst = os.lstat(live_path) - - # Since os.lstat() inside TarFile.gettarinfo() can trigger a - # UnicodeEncodeError when python has something other than utf_8 - # return from sys.getfilesystemencoding() (as in bug #388773), - # we implement the needed functionality here, using the result - # of our successful lstat call. An alternative to this would be - # to pass in the fileobj argument to TarFile.gettarinfo(), so - # that it could use fstat instead of lstat. However, that would - # have the unwanted effect of dereferencing symlinks. - - tarinfo = tar.tarinfo() - tarinfo.name = arcname - tarinfo.mode = lst.st_mode - tarinfo.uid = lst.st_uid - tarinfo.gid = lst.st_gid - tarinfo.size = 0 - tarinfo.mtime = lst.st_mtime - tarinfo.linkname = "" - if stat.S_ISREG(lst.st_mode): - inode = (lst.st_ino, lst.st_dev) - if (lst.st_nlink > 1 and - inode in tar.inodes and - arcname != tar.inodes[inode]): - tarinfo.type = tarfile.LNKTYPE - tarinfo.linkname = tar.inodes[inode] - else: - tar.inodes[inode] = arcname - tarinfo.type = tarfile.REGTYPE - tarinfo.size = lst.st_size - elif stat.S_ISDIR(lst.st_mode): - tarinfo.type = tarfile.DIRTYPE - elif stat.S_ISLNK(lst.st_mode): - tarinfo.type = tarfile.SYMTYPE - tarinfo.linkname = os.readlink(live_path) - else: - continue - try: - tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0] - except KeyError: - pass - try: - tarinfo.gname = grp.getgrgid(tarinfo.gid)[0] - except KeyError: - pass - - if stat.S_ISREG(lst.st_mode): - if protect and protect(path): - # Create an empty file as a place holder in order to avoid - # potential collision-protect issues. - f = tempfile.TemporaryFile() - f.write(_unicode_encode( - "# empty file because --include-config=n " + \ - "when `quickpkg` was used\n")) - f.flush() - f.seek(0) - tarinfo.size = os.fstat(f.fileno()).st_size - tar.addfile(tarinfo, f) - f.close() - else: - path_bytes = _unicode_encode(path, - encoding=encoding, - errors='strict') - - if xattrs: - # Compatible with GNU tar, which saves the xattrs - # under the SCHILY.xattr namespace. - for k in xattr.list(path_bytes): - tarinfo.pax_headers['SCHILY.xattr.' + - _unicode_decode(k)] = _unicode_decode( - xattr.get(path_bytes, _unicode_encode(k))) - - with open(path_bytes, 'rb') as f: - tar.addfile(tarinfo, f) - - else: - tar.addfile(tarinfo) - if onProgress: - onProgress(maxval, curval) -- cgit v1.2.3-65-gdbad