path: root/pym/portage/dbapi/porttree.py

# Copyright 1998-2010 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2

__all__ = [
	"close_portdbapi_caches", "FetchlistDict", "portagetree", "portdbapi"
]

import portage
portage.proxy.lazyimport.lazyimport(globals(),
	'portage.checksum',
	'portage.data:portage_gid,secpass',
	'portage.dbapi.dep_expand:dep_expand',
	'portage.dep:dep_getkey,match_from_list,use_reduce',
	'portage.env.loaders:KeyValuePairFileLoader',
	'portage.package.ebuild.doebuild:doebuild',
	'portage.util:ensure_dirs,shlex_split,writemsg,writemsg_level',
	'portage.util.listdir:listdir',
	'portage.versions:best,catpkgsplit,_pkgsplit@pkgsplit,ver_regexp',
)

from portage.cache.cache_errors import CacheError
from portage.cache.mappings import Mapping
from portage.const import REPO_NAME_LOC
from portage.dbapi import dbapi
from portage.exception import PortageException, \
	FileNotFound, InvalidDependString, InvalidPackageName
from portage.localization import _
from portage.manifest import Manifest

from portage import eclass_cache, auxdbkeys, \
	eapi_is_supported, dep_check, \
	_eapi_is_deprecated
from portage import os
from portage import _encodings
from portage import _unicode_encode
from portage import OrderedDict
from _emerge.EbuildMetadataPhase import EbuildMetadataPhase
from _emerge.PollScheduler import PollScheduler

import os as _os
import codecs
import logging
import stat
import sys
import warnings

if sys.hexversion >= 0x3000000:
	basestring = str
	long = int

class _repo_info(object):
	__slots__ = ('name', 'path', 'eclass_db', 'portdir', 'portdir_overlay')
	def __init__(self, name, path, eclass_db):
		self.name = name
		self.path = path
		self.eclass_db = eclass_db
		self.portdir = eclass_db.porttrees[0]
		self.portdir_overlay = ' '.join(eclass_db.porttrees[1:])

class portdbapi(dbapi):
	"""this tree will scan a portage directory located at root (passed to init)"""
	portdbapi_instances = []
	_use_mutable = True

	@property
	def _categories(self):
		return self.settings.categories

	def __init__(self, _unused_param=None, mysettings=None):
		"""
		@param _unused_param: deprecated, use mysettings['PORTDIR'] instead
		@type _unused_param: None
		@param mysettings: an immutable config instance
		@type mysettings: portage.config
		"""
		portdbapi.portdbapi_instances.append(self)

		from portage import config
		if mysettings:
			self.settings = mysettings
		else:
			from portage import settings
			self.settings = config(clone=settings)

		porttree_root = self.settings['PORTDIR']

		if _unused_param is not None and _unused_param != porttree_root:
			warnings.warn("The first parameter of the " + \
				"portage.dbapi.porttree.portdbapi" + \
				" constructor is now unused. " + \
				"mysettings['PORTDIR'] will be used instead.",
				DeprecationWarning, stacklevel=2)

		# This is strictly for use in aux_get() doebuild calls when metadata
		# is generated by the depend phase.  It's safest to use a clone for
		# this purpose because doebuild makes many changes to the config
		# instance that is passed in.
		self.doebuild_settings = config(clone=self.settings)
		self.depcachedir = os.path.realpath(self.settings.depcachedir)

		if os.environ.get("SANDBOX_ON") == "1":
			# Make api consumers exempt from sandbox violations
			# when doing metadata cache updates.
			sandbox_write = os.environ.get("SANDBOX_WRITE", "").split(":")
			if self.depcachedir not in sandbox_write:
				sandbox_write.append(self.depcachedir)
				os.environ["SANDBOX_WRITE"] = \
					":".join(filter(None, sandbox_write))

		porttrees = [os.path.realpath(porttree_root)]
		porttrees.extend(os.path.realpath(x) for x in \
			shlex_split(self.settings.get('PORTDIR_OVERLAY', '')))
		treemap = {}
		repository_map = {}
		self.treemap = treemap
		self._repository_map = repository_map
		identically_named_paths = {}
		for path in porttrees:
			if path in repository_map:
				continue
			repo_name_path = os.path.join(path, REPO_NAME_LOC)
			try:
				repo_name = codecs.open(
					_unicode_encode(repo_name_path,
					encoding=_encodings['fs'], errors='strict'),
					mode='r', encoding=_encodings['repo.content'],
					errors='replace').readline().strip()
			except EnvironmentError:
				# warn about missing repo_name at some other time, since we
				# don't want to see a warning every time the portage module is
				# imported.
				pass
			else:
				identically_named_path = treemap.get(repo_name)
				if identically_named_path is not None:
					# The earlier one is discarded.
					del repository_map[identically_named_path]
					identically_named_paths[identically_named_path] = repo_name
					if identically_named_path == porttrees[0]:
						# Found another repo with the same name as
						# $PORTDIR, so update porttrees[0] to match.
						porttrees[0] = path
				treemap[repo_name] = path
				repository_map[path] = repo_name

		# Ensure that each repo_name is unique. Later paths override
		# earlier ones that correspond to the same name.
		porttrees = [x for x in porttrees if x not in identically_named_paths]
		ignored_map = {}
		for path, repo_name in identically_named_paths.items():
			ignored_map.setdefault(repo_name, []).append(path)
		self._ignored_repos = tuple((repo_name, tuple(paths)) \
			for repo_name, paths in ignored_map.items())

		self.porttrees = porttrees
		porttree_root = porttrees[0]
		self.porttree_root = porttree_root

		self.eclassdb = eclass_cache.cache(porttree_root)

		# This is used as sanity check for aux_get(). If there is no
		# root eclass dir, we assume that PORTDIR is invalid or
		# missing. This check allows aux_get() to detect a missing
		# portage tree and return early by raising a KeyError.
		self._have_root_eclass_dir = os.path.isdir(
			os.path.join(self.porttree_root, "eclass"))

		self.metadbmodule = self.settings.load_best_module("portdbapi.metadbmodule")

		#if the portdbapi is "frozen", then we assume that we can cache everything (that no updates to it are happening)
		self.xcache = {}
		self.frozen = 0

		self._repo_info = {}
		eclass_dbs = {porttree_root : self.eclassdb}
		local_repo_configs = self.settings._local_repo_configs
		default_loc_repo_config = None
		repo_aliases = {}
		if local_repo_configs is not None:
			default_loc_repo_config = local_repo_configs.get('DEFAULT')
			for repo_name, loc_repo_conf in local_repo_configs.items():
				if loc_repo_conf.aliases is not None:
					for alias in loc_repo_conf.aliases:
						overridden_alias = repo_aliases.get(alias)
						if overridden_alias is not None:
							writemsg_level(_("!!! Alias '%s' " \
								"created for '%s' overrides " \
								"'%s' alias in " \
								"'%s'\n") % (alias, repo_name,
								overridden_alias,
								self.settings._local_repo_conf_path),
								level=logging.WARNING, noiselevel=-1)
						repo_aliases[alias] = repo_name

		for path in self.porttrees:
			if path in self._repo_info:
				continue

			repo_name = self._repository_map.get(path)

			loc_repo_conf = None
			if local_repo_configs is not None:
				if repo_name is not None:
					loc_repo_conf = local_repo_configs.get(repo_name)
				if loc_repo_conf is None:
					loc_repo_conf = default_loc_repo_config

			layout_filename = os.path.join(path, "metadata/layout.conf")
			layout_file = KeyValuePairFileLoader(layout_filename, None, None)
			layout_data, layout_errors = layout_file.load()
			porttrees = []

			masters = None
			if loc_repo_conf is not None and \
				loc_repo_conf.masters is not None:
				masters = loc_repo_conf.masters
			else:
				masters = layout_data.get('masters', '').split()

			for master_name in masters:
				master_name = repo_aliases.get(master_name, master_name)
				master_path = self.treemap.get(master_name)
				if master_path is None:
					writemsg_level(_("Unavailable repository '%s' " \
						"referenced by masters entry in '%s'\n") % \
						(master_name, layout_filename),
						level=logging.ERROR, noiselevel=-1)
				else:
					porttrees.append(master_path)

			if not porttrees and path != porttree_root:
				# Make PORTDIR the default master, but only if our
				# heuristics suggest that it's necessary.
				profiles_desc = os.path.join(path, 'profiles', 'profiles.desc')
				eclass_dir = os.path.join(path, 'eclass')
				if not os.path.isfile(profiles_desc) or \
					not os.path.isdir(eclass_dir):
					porttrees.append(porttree_root)

			porttrees.append(path)

			if loc_repo_conf is not None and \
					loc_repo_conf.eclass_overrides is not None:
					for other_name in loc_repo_conf.eclass_overrides:
						other_path = self.treemap.get(other_name)
						if other_path is None:
							writemsg_level(_("Unavailable repository '%s' " \
								"referenced by eclass-overrides entry in " \
								"'%s'\n") % (other_name,
								self.settings._local_repo_conf_path),
								level=logging.ERROR, noiselevel=-1)
							continue
						porttrees.append(other_path)

			eclass_db = None
			for porttree in porttrees:
				tree_db = eclass_dbs.get(porttree)
				if tree_db is None:
					tree_db = eclass_cache.cache(porttree)
					eclass_dbs[porttree] = tree_db
				if eclass_db is None:
					eclass_db = tree_db.copy()
				else:
					eclass_db.append(tree_db)

			self._repo_info[path] = _repo_info(repo_name, path, eclass_db)

		self.auxdbmodule = self.settings.load_best_module("portdbapi.auxdbmodule")
		self.auxdb = {}
		self._pregen_auxdb = {}
		self._init_cache_dirs()
		depcachedir_w_ok = os.access(self.depcachedir, os.W_OK)
		cache_kwargs = {
			'gid'     : portage_gid,
			'perms'   : 0o664
		}

		if secpass < 1:
			# portage_gid is irrelevant, so just obey umask
			cache_kwargs['gid']   = -1
			cache_kwargs['perms'] = -1

		# XXX: REMOVE THIS ONCE UNUSED_0 IS YANKED FROM auxdbkeys
		# ~harring
		filtered_auxdbkeys = [x for x in auxdbkeys if not x.startswith("UNUSED_0")]
		filtered_auxdbkeys.sort()
		from portage.cache import metadata_overlay, volatile
		if not depcachedir_w_ok:
			for x in self.porttrees:
				db_ro = self.auxdbmodule(self.depcachedir, x,
					filtered_auxdbkeys, gid=portage_gid, readonly=True)
				self.auxdb[x] = metadata_overlay.database(
					self.depcachedir, x, filtered_auxdbkeys,
					gid=portage_gid, db_rw=volatile.database,
					db_ro=db_ro)
		else:
			for x in self.porttrees:
				if x in self.auxdb:
					continue
				# location, label, auxdbkeys
				self.auxdb[x] = self.auxdbmodule(
					self.depcachedir, x, filtered_auxdbkeys, **cache_kwargs)
				if self.auxdbmodule is metadata_overlay.database:
					self.auxdb[x].db_ro.ec = self._repo_info[x].eclass_db
		if "metadata-transfer" not in self.settings.features:
			for x in self.porttrees:
				if x in self._pregen_auxdb:
					continue
				if os.path.isdir(os.path.join(x, "metadata", "cache")):
					self._pregen_auxdb[x] = self.metadbmodule(
						x, "metadata/cache", filtered_auxdbkeys, readonly=True)
					try:
						self._pregen_auxdb[x].ec = self._repo_info[x].eclass_db
					except AttributeError:
						pass
		# Selectively cache metadata in order to optimize dep matching.
		self._aux_cache_keys = set(
			["DEPEND", "EAPI", "INHERITED", "IUSE", "KEYWORDS", "LICENSE",
			"PDEPEND", "PROPERTIES", "PROVIDE", "RDEPEND", "repository",
			"RESTRICT", "SLOT", "DEFINED_PHASES", "REQUIRED_USE"])

		self._aux_cache = {}
		self._broken_ebuilds = set()

	def _init_cache_dirs(self):
		"""Create /var/cache/edb/dep and adjust permissions for the portage
		group."""

		dirmode  = 0o2070
		filemode =   0o60
		modemask =    0o2

		try:
			ensure_dirs(self.depcachedir, gid=portage_gid,
				mode=dirmode, mask=modemask)
		except PortageException as e:
			pass

	def close_caches(self):
		if not hasattr(self, "auxdb"):
			# unhandled exception thrown from constructor
			return
		for x in self.auxdb:
			self.auxdb[x].sync()
		self.auxdb.clear()

	def flush_cache(self):
		for x in self.auxdb.values():
			x.sync()

	def findLicensePath(self, license_name):
		mytrees = self.porttrees[:]
		mytrees.reverse()
		for x in mytrees:
			license_path = os.path.join(x, "licenses", license_name)
			if os.access(license_path, os.R_OK):
				return license_path
		return None

	def findname(self,mycpv, mytree = None, myrepo = None):
		return self.findname2(mycpv, mytree, myrepo)[0]

	def getRepositoryPath(self, repository_id):
		"""
		This function is required for GLEP 42 compliance; given a valid repository ID
		it must return a path to the repository
		TreeMap = { id:path }
		"""
		if repository_id in self.treemap:
			return self.treemap[repository_id]
		return None

	def getRepositoryName(self, canonical_repo_path):
		"""
		This is the inverse of getRepositoryPath().
		@param canonical_repo_path: the canonical path of a repository, as
			resolved by os.path.realpath()
		@type canonical_repo_path: String
		@returns: The repo_name for the corresponding repository, or None
			if the path does not correspond a known repository
		@rtype: String or None
		"""
		return self._repository_map.get(canonical_repo_path)

	def getRepositories(self):
		"""
		This function is required for GLEP 42 compliance; it will return a list of
		repository IDs
		TreeMap = {id: path}
		"""
		return [k for k in self.treemap if k]

	def findname2(self, mycpv, mytree=None, myrepo = None):
		""" 
		Returns the location of the CPV, and what overlay it was in.
		Searches overlays first, then PORTDIR; this allows us to return the first
		matching file.  As opposed to starting in portdir and then doing overlays
		second, we would have to exhaustively search the overlays until we found
		the file we wanted.
		If myrepo is not None it will find packages from this repository(overlay)
		"""
		if not mycpv:
			return (None, 0)

		if myrepo:
			if myrepo in self.treemap:
				mytree = self.treemap[myrepo]
			else:
				return (None, 0)
		
		mysplit = mycpv.split("/")
		psplit = pkgsplit(mysplit[1])
		if psplit is None or len(mysplit) != 2:
			raise InvalidPackageName(mycpv)

		# For optimal performace in this hot spot, we do manual unicode
		# handling here instead of using the wrapped os module.
		encoding = _encodings['fs']
		errors = 'strict'

		if mytree:
			mytrees = [mytree]
		else:
			mytrees = self.porttrees[:]
			mytrees.reverse()

		relative_path = mysplit[0] + _os.sep + psplit[0] + _os.sep + \
			mysplit[1] + ".ebuild"

		for x in mytrees:
			filename = x + _os.sep + relative_path
			if _os.access(_unicode_encode(filename,
				encoding=encoding, errors=errors), _os.R_OK):
				return (filename, x)
		return (None, 0)

	def _metadata_process(self, cpv, ebuild_path, repo_path):
		"""
		Create an EbuildMetadataPhase instance to generate metadata for the
		give ebuild.
		@rtype: EbuildMetadataPhase
		@returns: A new EbuildMetadataPhase instance, or None if the
			metadata cache is already valid.
		"""
		metadata, st, emtime = self._pull_valid_cache(cpv, ebuild_path, repo_path)
		if metadata is not None:
			return None

		process = EbuildMetadataPhase(cpv=cpv, ebuild_path=ebuild_path,
			ebuild_mtime=emtime, metadata_callback=self._metadata_callback,
			portdb=self, repo_path=repo_path, settings=self.doebuild_settings)
		return process

	def _metadata_callback(self, cpv, ebuild_path, repo_path, metadata, mtime):

		i = metadata
		if hasattr(metadata, "items"):
			i = iter(metadata.items())
		metadata = dict(i)

		if metadata.get("INHERITED", False):
			metadata["_eclasses_"] = self._repo_info[repo_path
				].eclass_db.get_eclass_data(metadata["INHERITED"].split())
		else:
			metadata["_eclasses_"] = {}

		metadata.pop("INHERITED", None)
		metadata["_mtime_"] = mtime

		eapi = metadata.get("EAPI")
		if not eapi or not eapi.strip():
			eapi = "0"
			metadata["EAPI"] = eapi
		if not eapi_is_supported(eapi):
			for k in set(metadata).difference(("_mtime_", "_eclasses_")):
				metadata[k] = ""
			metadata["EAPI"] = "-" + eapi.lstrip("-")

		self.auxdb[repo_path][cpv] = metadata
		return metadata

	def _pull_valid_cache(self, cpv, ebuild_path, repo_path):
		try:
			# Don't use unicode-wrapped os module, for better performance.
			st = _os.stat(_unicode_encode(ebuild_path,
				encoding=_encodings['fs'], errors='strict'))
			emtime = st[stat.ST_MTIME]
		except OSError:
			writemsg(_("!!! aux_get(): ebuild for " \
				"'%s' does not exist at:\n") % (cpv,), noiselevel=-1)
			writemsg("!!!            %s\n" % ebuild_path, noiselevel=-1)
			raise KeyError(cpv)

		# Pull pre-generated metadata from the metadata/cache/
		# directory if it exists and is valid, otherwise fall
		# back to the normal writable cache.
		auxdbs = []
		pregen_auxdb = self._pregen_auxdb.get(repo_path)
		if pregen_auxdb is not None:
			auxdbs.append(pregen_auxdb)
		auxdbs.append(self.auxdb[repo_path])
		eclass_db = self._repo_info[repo_path].eclass_db

		doregen = True
		for auxdb in auxdbs:
			try:
				metadata = auxdb[cpv]
			except KeyError:
				pass
			except CacheError:
				if auxdb is not pregen_auxdb:
					try:
						del auxdb[cpv]
					except KeyError:
						pass
					except CacheError:
						pass
			else:
				eapi = metadata.get('EAPI', '').strip()
				if not eapi:
					eapi = '0'
				if not (eapi[:1] == '-' and eapi_is_supported(eapi[1:])) and \
					emtime == metadata['_mtime_'] and \
					eclass_db.is_eclass_data_valid(metadata['_eclasses_']):
					doregen = False

			if not doregen:
				break

		if doregen:
			metadata = None

		return (metadata, st, emtime)

	def aux_get(self, mycpv, mylist, mytree=None, myrepo=None):
		"stub code for returning auxilliary db information, such as SLOT, DEPEND, etc."
		'input: "sys-apps/foo-1.0",["SLOT","DEPEND","HOMEPAGE"]'
		'return: ["0",">=sys-libs/bar-1.0","http://www.foo.com"] or raise KeyError if error'
		cache_me = False
		if myrepo:
			if myrepo in self.treemap:
				mytree = self.treemap[myrepo]
			else:
				raise KeyError(myrepo)
				
		if not mytree:
			cache_me = True
		if not mytree and not self._known_keys.intersection(
			mylist).difference(self._aux_cache_keys):
			aux_cache = self._aux_cache.get(mycpv)
			if aux_cache is not None:
				return [aux_cache.get(x, "") for x in mylist]
			cache_me = True
		global auxdbkeys, auxdbkeylen
		try:
			cat, pkg = mycpv.split("/", 1)
		except ValueError:
			# Missing slash. Can't find ebuild so raise KeyError.
			raise KeyError(mycpv)

		myebuild, mylocation = self.findname2(mycpv, mytree)

		if not myebuild:
			writemsg("!!! aux_get(): %s\n" % \
				_("ebuild not found for '%s'") % mycpv, noiselevel=1)
			raise KeyError(mycpv)

		mydata, st, emtime = self._pull_valid_cache(mycpv, myebuild, mylocation)
		doregen = mydata is None

		if doregen:
			if myebuild in self._broken_ebuilds:
				raise KeyError(mycpv)
			if not self._have_root_eclass_dir:
				raise KeyError(mycpv)

			self.doebuild_settings.setcpv(mycpv)
			eapi = None

			if eapi is None and \
				'parse-eapi-ebuild-head' in self.doebuild_settings.features:
				eapi = portage._parse_eapi_ebuild_head(codecs.open(
					_unicode_encode(myebuild,
					encoding=_encodings['fs'], errors='strict'),
					mode='r', encoding=_encodings['repo.content'],
					errors='replace'))

			if eapi is not None:
				self.doebuild_settings.configdict['pkg']['EAPI'] = eapi

			if eapi is not None and not portage.eapi_is_supported(eapi):
				mydata = self._metadata_callback(
					mycpv, myebuild, mylocation, {'EAPI':eapi}, emtime)
			else:
				proc = EbuildMetadataPhase(cpv=mycpv, ebuild_path=myebuild,
					ebuild_mtime=emtime,
					metadata_callback=self._metadata_callback, portdb=self,
					repo_path=mylocation,
					scheduler=PollScheduler().sched_iface,
					settings=self.doebuild_settings)

				proc.start()
				proc.wait()

				if proc.returncode != os.EX_OK:
					self._broken_ebuilds.add(myebuild)
					raise KeyError(mycpv)

				mydata = proc.metadata

		# do we have a origin repository name for the current package
		mydata["repository"] = self._repository_map.get(mylocation, "")

		mydata["INHERITED"] = ' '.join(mydata.get("_eclasses_", []))
		mydata["_mtime_"] = st[stat.ST_MTIME]

		eapi = mydata.get("EAPI")
		if not eapi:
			eapi = "0"
			mydata["EAPI"] = eapi
		if not eapi_is_supported(eapi):
			for k in set(mydata).difference(("_mtime_", "_eclasses_")):
				mydata[k] = ""
			mydata["EAPI"] = "-" + eapi.lstrip("-")

		#finally, we look at our internal cache entry and return the requested data.
		returnme = [mydata.get(x, "") for x in mylist]

		if cache_me:
			aux_cache = {}
			for x in self._aux_cache_keys:
				aux_cache[x] = mydata.get(x, "")
			self._aux_cache[mycpv] = aux_cache

		return returnme

	def getFetchMap(self, mypkg, useflags=None, mytree=None):
		"""
		Get the SRC_URI metadata as a dict which maps each file name to a
		set of alternative URIs.

		@param mypkg: cpv for an ebuild
		@type mypkg: String
		@param useflags: a collection of enabled USE flags, for evaluation of
			conditionals
		@type useflags: set, or None to enable all conditionals
		@param mytree: The canonical path of the tree in which the ebuild
			is located, or None for automatic lookup
		@type mypkg: String
		@returns: A dict which maps each file name to a set of alternative
			URIs.
		@rtype: dict
		"""

		try:
			eapi, myuris = self.aux_get(mypkg,
				["EAPI", "SRC_URI"], mytree=mytree)
		except KeyError:
			# Convert this to an InvalidDependString exception since callers
			# already handle it.
			raise portage.exception.InvalidDependString(
				"getFetchMap(): aux_get() error reading "+mypkg+"; aborting.")

		if not eapi_is_supported(eapi):
			# Convert this to an InvalidDependString exception
			# since callers already handle it.
			raise portage.exception.InvalidDependString(
				"getFetchMap(): '%s' has unsupported EAPI: '%s'" % \
				(mypkg, eapi.lstrip("-")))

		return _parse_uri_map(mypkg, {'EAPI':eapi,'SRC_URI':myuris},
			use=useflags)

	def getfetchsizes(self, mypkg, useflags=None, debug=0, myrepo=None):
		# returns a filename:size dictionnary of remaining downloads
		myebuild = self.findname(mypkg, myrepo=myrepo)
		if myebuild is None:
			raise AssertionError(_("ebuild not found for '%s'") % mypkg)
		pkgdir = os.path.dirname(myebuild)
		mf = Manifest(pkgdir, self.settings["DISTDIR"])
		checksums = mf.getDigests()
		if not checksums:
			if debug: 
				writemsg(_("[empty/missing/bad digest]: %s\n") % (mypkg,))
			return {}
		filesdict={}
		myfiles = self.getFetchMap(mypkg, useflags=useflags)
		#XXX: maybe this should be improved: take partial downloads
		# into account? check checksums?
		for myfile in myfiles:
			try:
				fetch_size = int(checksums[myfile]["size"])
			except (KeyError, ValueError):
				if debug:
					writemsg(_("[bad digest]: missing %(file)s for %(pkg)s\n") % {"file":myfile, "pkg":mypkg})
				continue
			file_path = os.path.join(self.settings["DISTDIR"], myfile)
			mystat = None
			try:
				mystat = os.stat(file_path)
			except OSError as e:
				pass
			if mystat is None:
				existing_size = 0
				ro_distdirs = self.settings.get("PORTAGE_RO_DISTDIRS")
				if ro_distdirs is not None:
					for x in shlex_split(ro_distdirs):
						try:
							mystat = os.stat(os.path.join(x, myfile))
						except OSError:
							pass
						else:
							if mystat.st_size == fetch_size:
								existing_size = fetch_size
								break
			else:
				existing_size = mystat.st_size
			remaining_size = fetch_size - existing_size
			if remaining_size > 0:
				# Assume the download is resumable.
				filesdict[myfile] = remaining_size
			elif remaining_size < 0:
				# The existing file is too large and therefore corrupt.
				filesdict[myfile] = int(checksums[myfile]["size"])
		return filesdict

	def fetch_check(self, mypkg, useflags=None, mysettings=None, all=False, myrepo=None):
		if all:
			useflags = None
		elif useflags is None:
			if mysettings:
				useflags = mysettings["USE"].split()
		if myrepo:
			if myrepo in self.treemap:
				mytree = self.treemap[myrepo]
			else:
				return False
		else:
			mytree = None

		myfiles = self.getFetchMap(mypkg, useflags=useflags, mytree=mytree)
		myebuild = self.findname(mypkg, myrepo=myrepo)
		if myebuild is None:
			raise AssertionError(_("ebuild not found for '%s'") % mypkg)
		pkgdir = os.path.dirname(myebuild)
		mf = Manifest(pkgdir, self.settings["DISTDIR"])
		mysums = mf.getDigests()

		failures = {}
		for x in myfiles:
			if not mysums or x not in mysums:
				ok     = False
				reason = _("digest missing")
			else:
				try:
					ok, reason = portage.checksum.verify_all(
						os.path.join(self.settings["DISTDIR"], x), mysums[x])
				except FileNotFound as e:
					ok = False
					reason = _("File Not Found: '%s'") % (e,)
			if not ok:
				failures[x] = reason
		if failures:
			return False
		return True

	def cpv_exists(self, mykey):
		"Tells us whether an actual ebuild exists on disk (no masking)"
		cps2 = mykey.split("/")
		cps = catpkgsplit(mykey, silent=0)
		if not cps:
			#invalid cat/pkg-v
			return 0
		if self.findname(cps[0] + "/" + cps2[1]):
			return 1
		else:
			return 0

	def cp_all(self, categories=None, trees=None):
		"""
		This returns a list of all keys in our tree or trees
		@param categories: optional list of categories to search or 
			defaults to self.settings.categories
		@param trees: optional list of trees to search the categories in or
			defaults to self.porttrees
		@rtype list of [cat/pkg,...]
		"""
		d = {}
		if categories is None:
			categories = self.settings.categories
		if trees is None:
			trees = self.porttrees
		for x in categories:
			for oroot in trees:
				for y in listdir(oroot+"/"+x, EmptyOnError=1, ignorecvs=1, dirsonly=1):
					if not self._pkg_dir_name_re.match(y) or \
						y == "CVS":
						continue
					d[x+"/"+y] = None
		l = list(d)
		l.sort()
		return l

	def cp_list(self, mycp, use_cache=1, mytree=None):
		if self.frozen and mytree is None:
			cachelist = self.xcache["cp-list"].get(mycp)
			if cachelist is not None:
				# Try to propagate this to the match-all cache here for
				# repoman since he uses separate match-all caches for each
				# profile (due to old-style virtuals). Do not propagate
				# old-style virtuals since cp_list() doesn't expand them.
				if not (not cachelist and mycp.startswith("virtual/")):
					self.xcache["match-all"][mycp] = cachelist
				return cachelist[:]
		mysplit = mycp.split("/")
		invalid_category = mysplit[0] not in self._categories
		d={}
		if mytree is not None:
			if isinstance(mytree, basestring):
				mytrees = [mytree]
			else:
				# assume it's iterable
				mytrees = mytree
		else:
			mytrees = self.porttrees
		for oroot in mytrees:
			try:
				file_list = os.listdir(os.path.join(oroot, mycp))
			except OSError:
				continue
			for x in file_list:
				pf = None
				if x[-7:] == '.ebuild':
					pf = x[:-7]

				if pf is not None:
					ps = pkgsplit(pf)
					if not ps:
						writemsg(_("\nInvalid ebuild name: %s\n") % \
							os.path.join(oroot, mycp, x), noiselevel=-1)
						continue
					if ps[0] != mysplit[1]:
						writemsg(_("\nInvalid ebuild name: %s\n") % \
							os.path.join(oroot, mycp, x), noiselevel=-1)
						continue
					ver_match = ver_regexp.match("-".join(ps[1:]))
					if ver_match is None or not ver_match.groups():
						writemsg(_("\nInvalid ebuild version: %s\n") % \
							os.path.join(oroot, mycp, x), noiselevel=-1)
						continue
					d[mysplit[0]+"/"+pf] = None
		if invalid_category and d:
			writemsg(_("\n!!! '%s' has a category that is not listed in " \
				"%setc/portage/categories\n") % \
				(mycp, self.settings["PORTAGE_CONFIGROOT"]), noiselevel=-1)
			mylist = []
		else:
			mylist = list(d)
		# Always sort in ascending order here since it's handy
		# and the result can be easily cached and reused.
		self._cpv_sort_ascending(mylist)
		if self.frozen and mytree is None:
			cachelist = mylist[:]
			self.xcache["cp-list"][mycp] = cachelist
			# Do not propagate old-style virtuals since
			# cp_list() doesn't expand them.
			if not (not cachelist and mycp.startswith("virtual/")):
				self.xcache["match-all"][mycp] = cachelist
		return mylist

	def freeze(self):
		for x in "bestmatch-visible", "cp-list", "list-visible", "match-all", \
			"match-visible", "minimum-all", "minimum-visible":
			self.xcache[x]={}
		self.frozen=1

	def melt(self):
		self.xcache = {}
		self.frozen = 0

	def xmatch(self,level,origdep,mydep=None,mykey=None,mylist=None):
		"caching match function; very trick stuff"
		#if no updates are being made to the tree, we can consult our xcache...
		if self.frozen:
			try:
				return self.xcache[level][origdep][:]
			except KeyError:
				pass

		if not mydep:
			#this stuff only runs on first call of xmatch()
			#create mydep, mykey from origdep
			mydep = dep_expand(origdep, mydb=self, settings=self.settings)
			mykey = mydep.cp

		if level == "list-visible":
			#a list of all visible packages, not called directly (just by xmatch())
			#myval = self.visible(self.cp_list(mykey))

			myval = self.gvisible(self.visible(self.cp_list(mykey)))
		elif level == "minimum-all":
			# Find the minimum matching version. This is optimized to
			# minimize the number of metadata accesses (improves performance
			# especially in cases where metadata needs to be generated).
			cpv_iter = iter(self.cp_list(mykey))
			if mydep != mykey:
				cpv_iter = self._iter_match(mydep, cpv_iter)
			try:
				myval = next(cpv_iter)
			except StopIteration:
				myval = ""

		elif level in ("minimum-visible", "bestmatch-visible"):
			# Find the minimum matching visible version. This is optimized to
			# minimize the number of metadata accesses (improves performance
			# especially in cases where metadata needs to be generated).
			if mydep == mykey:
				mylist = self.cp_list(mykey)
			else:
				mylist = match_from_list(mydep, self.cp_list(mykey, myrepo = mydep.repo))
			myval = ""
			settings = self.settings
			local_config = settings.local_config
			aux_keys = list(self._aux_cache_keys)
			if level == "minimum-visible":
				iterfunc = iter
			else:
				iterfunc = reversed
			for cpv in iterfunc(mylist):
				try:
					metadata = dict(zip(aux_keys,
						self.aux_get(cpv, aux_keys)))
				except KeyError:
					# ebuild masked by corruption
					continue
				if not eapi_is_supported(metadata["EAPI"]):
					continue
				if mydep.slot and mydep.slot != metadata["SLOT"]:
					continue
				if settings._getMissingKeywords(cpv, metadata):
					continue
				if settings._getMaskAtom(cpv, metadata):
					continue
				if settings._getProfileMaskAtom(cpv, metadata):
					continue
				if local_config:
					metadata["USE"] = ""
					if "?" in metadata["LICENSE"] or "?" in metadata["PROPERTIES"]:
						self.doebuild_settings.setcpv(cpv, mydb=metadata)
						metadata["USE"] = self.doebuild_settings.get("USE", "")
					try:
						if settings._getMissingLicenses(cpv, metadata):
							continue
						if settings._getMissingProperties(cpv, metadata):
							continue
					except InvalidDependString:
						continue
				if mydep.use:
					has_iuse = False
					for has_iuse in self._iter_match_use(mydep, [cpv]):
						break
					if not has_iuse:
						continue
				myval = cpv
				break
		elif level == "bestmatch-list":
			#dep match -- find best match but restrict search to sublist
			#no point in calling xmatch again since we're not caching list deps

			myval = best(list(self._iter_match(mydep, mylist)))
		elif level == "match-list":
			#dep match -- find all matches but restrict search to sublist (used in 2nd half of visible())

			myval = list(self._iter_match(mydep, mylist))
		elif level == "match-visible":
			#dep match -- find all visible matches
			#get all visible packages, then get the matching ones
			myval = list(self._iter_match(mydep,
				self.xmatch("list-visible", mykey, mydep=mykey, mykey=mykey), myrepo=mydep.repo))
		elif level == "match-all":
			#match *all* visible *and* masked packages
			if mydep == mykey:
				myval = self.cp_list(mykey)
			else:
				myval = list(self._iter_match(mydep, self.cp_list(mykey), myrepo = mydep.repo))
		else:
			raise AssertionError(
				"Invalid level argument: '%s'" % level)

		if self.frozen and (level not in ["match-list", "bestmatch-list"]):
			self.xcache[level][mydep] = myval
			if origdep and origdep != mydep:
				self.xcache[level][origdep] = myval
		return myval[:]

	def match(self, mydep, use_cache=1):
		return self.xmatch("match-visible", mydep)

	def visible(self, mylist):
		"""two functions in one.  Accepts a list of cpv values and uses the package.mask *and*
		packages file to remove invisible entries, returning remaining items.  This function assumes
		that all entries in mylist have the same category and package name."""
		if not mylist:
			return []

		db_keys = ["SLOT"]
		visible = []
		getMaskAtom = self.settings._getMaskAtom
		getProfileMaskAtom = self.settings._getProfileMaskAtom
		for cpv in mylist:
			try:
				metadata = dict(zip(db_keys, self.aux_get(cpv, db_keys)))
			except KeyError:
				# masked by corruption
				continue
			if not metadata["SLOT"]:
				continue
			if getMaskAtom(cpv, metadata):
				continue
			if getProfileMaskAtom(cpv, metadata):
				continue
			visible.append(cpv)
		return visible

	def gvisible(self,mylist):
		"strip out group-masked (not in current group) entries"

		if mylist is None:
			return []
		newlist=[]
		aux_keys = list(self._aux_cache_keys)
		metadata = {}
		local_config = self.settings.local_config
		chost = self.settings.get('CHOST', '')
		accept_chost = self.settings._accept_chost
		for mycpv in mylist:
			metadata.clear()
			try:
				metadata.update(zip(aux_keys, self.aux_get(mycpv, aux_keys)))
			except KeyError:
				continue
			except PortageException as e:
				writemsg("!!! Error: aux_get('%s', %s)\n" % (mycpv, aux_keys),
					noiselevel=-1)
				writemsg("!!! %s\n" % (e,), noiselevel=-1)
				del e
				continue
			eapi = metadata["EAPI"]
			if not eapi_is_supported(eapi):
				continue
			if _eapi_is_deprecated(eapi):
				continue
			if self.settings._getMissingKeywords(mycpv, metadata):
				continue
			if local_config:
				metadata['CHOST'] = chost
				if not accept_chost(mycpv, metadata):
					continue
				metadata["USE"] = ""
				if "?" in metadata["LICENSE"] or "?" in metadata["PROPERTIES"]:
					self.doebuild_settings.setcpv(mycpv, mydb=metadata)
					metadata['USE'] = self.doebuild_settings['PORTAGE_USE']
				try:
					if self.settings._getMissingLicenses(mycpv, metadata):
						continue
					if self.settings._getMissingProperties(mycpv, metadata):
						continue
				except InvalidDependString:
					continue
			newlist.append(mycpv)
		return newlist

def close_portdbapi_caches():
	for i in portdbapi.portdbapi_instances:
		i.close_caches()

portage.process.atexit_register(portage.portageexit)

class portagetree(object):
	def __init__(self, root=None, virtual=None, settings=None):
		"""
		Constructor for a PortageTree
		
		@param root: deprecated, defaults to settings['ROOT']
		@type root: String/Path
		@param virtual: UNUSED
		@type virtual: No Idea
		@param settings: Portage Configuration object (portage.settings)
		@type settings: Instance of portage.config
		"""

		if settings is None:
			settings = portage.settings
		self.settings = settings

		if root is not None and root != settings['ROOT']:
			warnings.warn("The root parameter of the " + \
				"portage.dbapi.porttree.portagetree" + \
				" constructor is now unused. Use " + \
				"settings['ROOT'] instead.",
				DeprecationWarning, stacklevel=2)

		self.portroot = settings["PORTDIR"]
		self.virtual = virtual
		self.dbapi = portdbapi(mysettings=settings)

	@property
	def root(self):
		warnings.warn("The root attribute of " + \
			"portage.dbapi.porttree.portagetree" + \
			" is deprecated. Use " + \
			"settings['ROOT'] instead.",
			DeprecationWarning, stacklevel=2)
		return self.settings['ROOT']

	def dep_bestmatch(self,mydep):
		"compatibility method"
		mymatch = self.dbapi.xmatch("bestmatch-visible",mydep)
		if mymatch is None:
			return ""
		return mymatch

	def dep_match(self,mydep):
		"compatibility method"
		mymatch = self.dbapi.xmatch("match-visible",mydep)
		if mymatch is None:
			return []
		return mymatch

	def exists_specific(self,cpv):
		return self.dbapi.cpv_exists(cpv)

	def getallnodes(self):
		"""new behavior: these are all *unmasked* nodes.  There may or may not be available
		masked package for nodes in this nodes list."""
		return self.dbapi.cp_all()

	def getname(self, pkgname):
		"returns file location for this particular package (DEPRECATED)"
		if not pkgname:
			return ""
		mysplit = pkgname.split("/")
		psplit = pkgsplit(mysplit[1])
		return "/".join([self.portroot, mysplit[0], psplit[0], mysplit[1]])+".ebuild"

	def depcheck(self, mycheck, use="yes", myusesplit=None):
		return dep_check(mycheck, self.dbapi, use=use, myuse=myusesplit)

	def getslot(self,mycatpkg):
		"Get a slot for a catpkg; assume it exists."
		myslot = ""
		try:
			myslot = self.dbapi.aux_get(mycatpkg, ["SLOT"])[0]
		except SystemExit as e:
			raise
		except Exception as e:
			pass
		return myslot

class FetchlistDict(Mapping):
	"""
	This provide a mapping interface to retrieve fetch lists. It's used
	to allow portage.manifest.Manifest to access fetch lists via a standard
	mapping interface rather than use the dbapi directly.
	"""
	def __init__(self, pkgdir, settings, mydbapi):
		"""pkgdir is a directory containing ebuilds and settings is passed into
		portdbapi.getfetchlist for __getitem__ calls."""
		self.pkgdir = pkgdir
		self.cp = os.sep.join(pkgdir.split(os.sep)[-2:])
		self.settings = settings
		self.mytree = os.path.realpath(os.path.dirname(os.path.dirname(pkgdir)))
		self.portdb = mydbapi

	def __getitem__(self, pkg_key):
		"""Returns the complete fetch list for a given package."""
		return list(self.portdb.getFetchMap(pkg_key, mytree=self.mytree))

	def __contains__(self, cpv):
		return cpv in self.__iter__()

	def has_key(self, pkg_key):
		"""Returns true if the given package exists within pkgdir."""
		warnings.warn("portage.dbapi.porttree.FetchlistDict.has_key() is "
			"deprecated, use the 'in' operator instead",
			DeprecationWarning, stacklevel=2)
		return pkg_key in self

	def __iter__(self):
		return iter(self.portdb.cp_list(self.cp, mytree=self.mytree))

	def __len__(self):
		"""This needs to be implemented in order to avoid
		infinite recursion in some cases."""
		return len(self.portdb.cp_list(self.cp, mytree=self.mytree))

	def keys(self):
		"""Returns keys for all packages within pkgdir"""
		return self.portdb.cp_list(self.cp, mytree=self.mytree)

	if sys.hexversion >= 0x3000000:
		keys = __iter__

def _parse_uri_map(cpv, metadata, use=None):

	myuris = use_reduce(metadata.get('SRC_URI', ''),
		uselist=use, matchall=(use is None),
		is_src_uri=True,
		eapi=metadata['EAPI'])

	uri_map = OrderedDict()

	myuris.reverse()
	while myuris:
		uri = myuris.pop()
		if myuris and myuris[-1] == "->":
			operator = myuris.pop()
			distfile = myuris.pop()
		else:
			distfile = os.path.basename(uri)
			if not distfile:
				raise portage.exception.InvalidDependString(
					("getFetchMap(): '%s' SRC_URI has no file " + \
					"name: '%s'") % (cpv, uri))

		uri_set = uri_map.get(distfile)
		if uri_set is None:
			uri_set = set()
			uri_map[distfile] = uri_set
		uri_set.add(uri)
		uri = None
		operator = None

	return uri_map