diff options
Diffstat (limited to 'pym/gentoolkit/eclean/search.py')
| -rw-r--r-- | pym/gentoolkit/eclean/search.py | 1122 |
1 files changed, 566 insertions, 556 deletions
diff --git a/pym/gentoolkit/eclean/search.py b/pym/gentoolkit/eclean/search.py index 8f6e52f..cb695c0 100644 --- a/pym/gentoolkit/eclean/search.py +++ b/pym/gentoolkit/eclean/search.py @@ -14,8 +14,12 @@ from portage.dep import Atom, use_reduce from portage.dep._slot_operator import strip_slots import gentoolkit.pprinter as pp -from gentoolkit.eclean.exclude import (exclDictMatchCP, exclDictExpand, - exclDictExpandPkgname, exclMatchFilename) +from gentoolkit.eclean.exclude import ( + exclDictMatchCP, + exclDictExpand, + exclDictExpandPkgname, + exclMatchFilename, +) # Misc. shortcuts to some portage stuff: @@ -23,7 +27,7 @@ port_settings = portage.settings pkgdir = port_settings["PKGDIR"] err = sys.stderr -deprecated_message=""""Deprecation Warning: Installed package: %s +deprecated_message = """"Deprecation Warning: Installed package: %s Is no longer in the tree or an installed overlay""" DEPRECATED = pp.warn(deprecated_message) @@ -31,574 +35,580 @@ debug_modules = [] def dprint(module, message): - if module in debug_modules: - print(message) + if module in debug_modules: + print(message) def get_distdir(): - """Returns DISTDIR if sane, else barfs.""" + """Returns DISTDIR if sane, else barfs.""" + + d = portage.settings["DISTDIR"] + if not os.path.isdir(d): + e = pp.error("%s does not appear to be a directory.\n" % d) + e += pp.error("Please set DISTDIR to a sane value.\n") + e += pp.error("(Check your make.conf file and environment).") + print(e, file=sys.stderr) + exit(1) + return d - d = portage.settings["DISTDIR"] - if not os.path.isdir(d): - e = pp.error("%s does not appear to be a directory.\n" % d) - e += pp.error("Please set DISTDIR to a sane value.\n") - e += pp.error("(Check your make.conf file and environment).") - print( e, file=sys.stderr) - exit(1) - return d distdir = get_distdir() class DistfilesSearch: - """ - - @param output: verbose output method or (lambda x: None) to turn off - @param vardb: defaults to portage.db[portage.root]["vartree"].dbapi - is overridden for testing. - @param portdb: defaults to portage.portdb and is overriden for testing. -""" - - def __init__(self, - output, - portdb=portage.portdb, - vardb=portage.db[portage.root]["vartree"].dbapi, - ): - self.vardb =vardb - self.portdb = portdb - self.output = output - self.installed_cpvs = None - - def findDistfiles(self, - exclude=None, - destructive=False, - fetch_restricted=False, - package_names=False, - time_limit=0, - size_limit=0, - _distdir=distdir, - deprecate=False, - extra_checks=() - ): - """Find all obsolete distfiles. - - XXX: what about cvs ebuilds? - I should install some to see where it goes... - - @param exclude: an exclusion dict as defined in - exclude.parseExcludeFile class. - @param destructive: boolean, defaults to False - @param fetch_restricted: boolean, defaults to False - @param package_names: boolean, defaults to False. - @param time_limit: integer time value as returned by parseTime() - @param size_limit: integer value of max. file size to keep or 0 to ignore. - @param _distdir: path to the distfiles dir being checked, defaults to portage. - @param deprecate: bool to control checking the clean dict. files for exclusion - - @rtype: dict - @return dict. of package files to clean i.e. {'cat/pkg-ver.tbz2': [filename],} - """ - if exclude is None: - exclude = {} - clean_me = {} - pkgs = {} - saved = {} - deprecated = {} - installed_included = False - # create a big CPV->SRC_URI dict of packages - # whose distfiles should be kept - if (not destructive) or fetch_restricted: - self.output("...non-destructive type search") - pkgs, _deprecated = self._non_destructive(destructive, fetch_restricted) - deprecated.update(_deprecated) - installed_included = True - if destructive: - self.output("...destructive type search: %d packages already found" %len(pkgs)) - pkgs, _deprecated = self._destructive(package_names, - exclude, pkgs, installed_included) - deprecated.update(_deprecated) - # gather the files to be cleaned - self.output("...checking limits for %d ebuild sources" - %len(pkgs)) - - checks = self._get_default_checks(size_limit, time_limit, exclude, destructive) - checks.extend(extra_checks) - clean_me = self._check_limits(_distdir, checks, clean_me) - # remove any protected files from the list - self.output("...removing protected sources from %s candidates to clean" - %len(clean_me)) - clean_me = self._remove_protected(pkgs, clean_me) - if not deprecate and len(exclude) and len(clean_me): - self.output("...checking final for exclusion from " +\ - "%s remaining candidates to clean" %len(clean_me)) - clean_me, saved = self._check_excludes(exclude, clean_me) - return clean_me, saved, deprecated - - -####################### begin _check_limits code block - - def _get_default_checks(self, size_limit, time_limit, excludes, destructive): - #checks =[(self._isreg_check_, "is_reg_check")] - checks =[self._isreg_check_] - if 'filenames' in excludes: - #checks.append((partial(self._filenames_check_, excludes), "Filenames_check")) - checks.append(partial(self._filenames_check_, excludes)) - else: - self.output(" - skipping exclude filenames check") - if size_limit: - #checks.append((partial(self._size_check_, size_limit), "size_check")) - checks.append(partial(self._size_check_, size_limit)) - else: - self.output(" - skipping size limit check") - if time_limit: - #print("time_limit = ", time_limit/1000000,"M sec") - #checks.append((partial(self._time_check_, time_limit), "time_check")) - checks.append(partial(self._time_check_, time_limit)) - else: - self.output(" - skipping time limit check") - if destructive: - self.output(" - skipping dot files check") - else: - checks.append(self._dotfile_check_) - return checks - - - def _check_limits(self, - _distdir, - checks, - clean_me=None - ): - """Checks files if they exceed size and/or time_limits, etc. - - To start with everything is considered dirty and is excluded - only if it matches some condition. - """ - if clean_me is None: - clean_me = {} - for file in os.listdir(_distdir): - filepath = os.path.join(_distdir, file) - try: - file_stat = os.lstat(filepath) - except EnvironmentError: - continue - is_dirty = False - #for check, check_name in checks: - for check in checks: - should_break, is_dirty = check(file_stat, file) - if should_break: - break - - if is_dirty: - #print( "%s Adding file to clean_list:" %check_name, file) - clean_me[file]=[filepath] - return clean_me - - @staticmethod - def _isreg_check_(file_stat, file): - """check if file is a regular file.""" - is_reg_file = stat.S_ISREG(file_stat[stat.ST_MODE]) - return not is_reg_file, is_reg_file - - @staticmethod - def _size_check_(size_limit, file_stat, file): - """checks if the file size exceeds the size_limit""" - if (file_stat[stat.ST_SIZE] >= size_limit): - #print( "size mismatch ", file, file_stat[stat.ST_SIZE]) - return True, False - return False, True - - @staticmethod - def _time_check_(time_limit, file_stat, file): - """checks if the file exceeds the time_limit - (think forward, not back, time keeps increasing)""" - if (file_stat[stat.ST_MTIME] >= time_limit): - #print( "time match too young ", file, file_stat[stat.ST_MTIME]/1000000,"M sec.") - return True, False - #print( "time match too old", file, file_stat[stat.ST_MTIME]/1000000,"M sec.") - return False, True - - @staticmethod - def _filenames_check_(exclude, file_stat, file): - """checks if the file matches an exclusion file listing""" - # Try to match file name directly - if file in exclude['filenames']: - return True, False - # See if file matches via regular expression matching - else: - file_match = False - for file_entry in exclude['filenames']: - if exclude['filenames'][file_entry].match(file): - file_match = True - break - if file_match: - #print( "filename match ", file) - return True, False - return False, True - - @staticmethod - def _dotfile_check_(file_stat, file): - """check if file is a regular file.""" - head, tail = os.path.split(file) - if tail: - is_dot_file = tail.startswith('.') - return is_dot_file, not is_dot_file - -####################### end _check_limits code block - - @staticmethod - def _remove_protected( - pkgs, - clean_me - ): - """Remove files owned by some protected packages. - - @returns packages to clean - @rtype: dictionary - """ - for cpv in pkgs: - uris = pkgs[cpv].split() - uris.reverse() - while uris: - uri = uris.pop() - if uris and uris[-1] == "->": - operator = uris.pop() # noqa - file = uris.pop() - else: - file = os.path.basename(uri) - if file in clean_me: - del clean_me[file] - # no need to waste IO time if there is nothing left to clean - if not len(clean_me): - return clean_me - return clean_me - - def _non_destructive(self, - destructive, - fetch_restricted, - pkgs_ = None, - hosts_cpvs=None - ): - """performs the non-destructive checks - - @param destructive: boolean - @param pkgs_: starting dictionary to add to - defaults to {}. - - @returns packages and thier SRC_URI's: {cpv: src_uri,} - @rtype: dictionary - """ - if pkgs_ is None: - pkgs = {} - else: - pkgs = pkgs_.copy() - deprecated = {} - # the following code block was split to optimize for speed - # list all CPV from portree (yeah, that takes time...) - self.output(" - getting complete ebuild list") - cpvs = set(self.portdb.cpv_all()) - installed_cpvs = set(self.vardb.cpv_all()) - # now add any installed cpv's that are not in the tree or overlays - cpvs.update(installed_cpvs) - # Add any installed cpvs from hosts on the network, if any - if hosts_cpvs: - cpvs.update(hosts_cpvs) - installed_cpvs.update(hosts_cpvs) - if fetch_restricted and destructive: - self.output(" - getting source file names " + - "for %d installed ebuilds" %len(installed_cpvs)) - pkgs, _deprecated = self._unrestricted(pkgs, installed_cpvs) - deprecated.update(_deprecated) - # remove the installed cpvs then check the remaining for fetch restiction - cpvs.difference_update(installed_cpvs) - self.output(" - getting fetch-restricted source file names " + - "for %d remaining ebuilds" %len(cpvs)) - pkgs, _deprecated = self._fetch_restricted(pkgs, cpvs) - deprecated.update(_deprecated) - # save the installed cpv list to re-use in _destructive() - self.installed_cpvs = installed_cpvs.copy() - else: - self.output(" - getting source file names " + - "for %d ebuilds" %len(cpvs)) - pkgs, _deprecated = self._unrestricted(pkgs, cpvs) - deprecated.update(_deprecated) - return pkgs, deprecated - - def _fetch_restricted(self, pkgs_, cpvs): - """perform fetch restricted non-destructive source - filename lookups - - @param pkgs_: starting dictionary to add to - @param cpvs: set of (cat/pkg-ver, ...) identifiers - - @return a new pkg dictionary - @rtype: dictionary - """ - if pkgs_ is None: - pkgs = {} - else: - pkgs = pkgs_.copy() - deprecated = {} - for cpv in cpvs: - # get SRC_URI and RESTRICT from aux_get - try: # main portdb - (src_uri,restrict) = \ - self.portdb.aux_get(cpv,["SRC_URI","RESTRICT"]) - # keep fetch-restricted check - # inside try so it is bypassed on KeyError - if 'fetch' in restrict: - pkgs[cpv] = src_uri - except KeyError: - try: # installed vardb - (src_uri,restrict) = \ - self.vardb.aux_get(cpv,["SRC_URI","RESTRICT"]) - deprecated[cpv] = src_uri - self.output(DEPRECATED %cpv) - # keep fetch-restricted check - # inside try so it is bypassed on KeyError - if 'fetch' in restrict: - pkgs[cpv] = src_uri - except KeyError: - self.output(" - Key Error looking up: " + cpv) - return pkgs, deprecated - - def _unrestricted(self, pkgs_, cpvs): - """Perform unrestricted source filenames lookups - - @param pkgs_: starting packages dictionary - @param cpvs: set of (cat/pkg-ver, ...) identifiers - - @return a new pkg dictionary - @rtype: dictionary - """ - if pkgs_ is None: - pkgs = {} - else: - pkgs = pkgs_.copy() - deprecated = {} - for cpv in cpvs: - # get SRC_URI from aux_get - try: - pkgs[cpv] = self.portdb.aux_get(cpv,["SRC_URI"])[0] - except KeyError: - try: # installed vardb - pkgs[cpv] = self.vardb.aux_get(cpv,["SRC_URI"])[0] - deprecated[cpv] = pkgs[cpv] - self.output(DEPRECATED %cpv) - except KeyError: - self.output(" - Key Error looking up: " + cpv) - return pkgs, deprecated - - def _destructive(self, - package_names, - exclude, - pkgs_=None, - installed_included=False - ): - """Builds on pkgs according to input options - - @param package_names: boolean - @param exclude: an exclusion dict as defined in - exclude.parseExcludeFile class. - @param pkgs: starting dictionary to add to - defaults to {}. - @param installed_included: bool. pkgs already - has the installed cpv's added. - - @returns pkgs: {cpv: src_uri,} - """ - if pkgs_ is None: - pkgs = {} - else: - pkgs = pkgs_.copy() - deprecated = {} - pkgset = set() - if not installed_included: - if not package_names: - # list all installed CPV's from vartree - #print( "_destructive: getting vardb.cpv_all") - if not self.installed_cpvs: - pkgset.update(self.vardb.cpv_all()) - else: - pkgset.update(self.installed_cpvs) - self.output(" - processing %s installed ebuilds" % len(pkgset)) - elif package_names: - # list all CPV's from portree for CP's in vartree - #print( "_destructive: getting vardb.cp_all") - cps = self.vardb.cp_all() - self.output(" - processing %s installed packages" % len(cps)) - for package in cps: - pkgset.update(self.portdb.cp_list(package)) - self.output(" - processing excluded") - excludes = self._get_excludes(exclude) - excludes_length = len(excludes) - dprint("excludes", "EXCLUDES LENGTH =%d" %excludes_length) - pkgset.update(excludes) - pkgs_done = set(list(pkgs)) - pkgset.difference_update(pkgs_done) - self.output( - " - (%d of %d total) additional excluded packages to get source filenames for" - %(len(pkgset), excludes_length)) - #self.output(" - processing %d ebuilds for filenames" %len(pkgset)) - pkgs, _deprecated = self._unrestricted(pkgs, pkgset) - deprecated.update(_deprecated) - #self.output(" - done...") - return pkgs, deprecated - - def _get_excludes(self, exclude): - """Expands the exclude dictionary into a set of - CPV's - - @param exclude: dictionary of exclusion categories, - packages to exclude from the cleaning - - @rtype: set - @return set of package cpv's - """ - pkgset = set() - for cp in exclDictExpand(exclude): - # add packages from the exclude file - dprint("excludes", "_GET_EXCLUDES, cp=" + \ - cp+", "+str(self.portdb.cp_list(cp))) - pkgset.update(self.portdb.cp_list(cp)) - return pkgset - - def _check_excludes(self, exclude, clean_me): - """Performs a last minute check on remaining filenames - to see if they should be protected. Since if the pkg-version - was deprecated it would not have been matched to a - source filename and removed. - - @param exclude: an exclusion dictionary - @param clean_me: the list of filenames for cleaning - - @rtype: dict of packages to clean - """ - saved = {} - pn_excludes = exclDictExpandPkgname(exclude) - dprint("excludes", "_check_excludes: made it here ;)") - if not pn_excludes: - return clean_me, saved - dprint("excludes", pn_excludes) - for key in list(clean_me): - if exclMatchFilename(pn_excludes, key): - saved[key] = clean_me[key] - del clean_me[key] - self.output(" ...Saved excluded package filename: " + key) - return clean_me, saved + """ + + @param output: verbose output method or (lambda x: None) to turn off + @param vardb: defaults to portage.db[portage.root]["vartree"].dbapi + is overridden for testing. + @param portdb: defaults to portage.portdb and is overriden for testing.""" + + def __init__( + self, + output, + portdb=portage.portdb, + vardb=portage.db[portage.root]["vartree"].dbapi, + ): + self.vardb = vardb + self.portdb = portdb + self.output = output + self.installed_cpvs = None + + def findDistfiles( + self, + exclude=None, + destructive=False, + fetch_restricted=False, + package_names=False, + time_limit=0, + size_limit=0, + _distdir=distdir, + deprecate=False, + extra_checks=(), + ): + """Find all obsolete distfiles. + + XXX: what about cvs ebuilds? + I should install some to see where it goes... + + @param exclude: an exclusion dict as defined in + exclude.parseExcludeFile class. + @param destructive: boolean, defaults to False + @param fetch_restricted: boolean, defaults to False + @param package_names: boolean, defaults to False. + @param time_limit: integer time value as returned by parseTime() + @param size_limit: integer value of max. file size to keep or 0 to ignore. + @param _distdir: path to the distfiles dir being checked, defaults to portage. + @param deprecate: bool to control checking the clean dict. files for exclusion + + @rtype: dict + @return dict. of package files to clean i.e. {'cat/pkg-ver.tbz2': [filename],} + """ + if exclude is None: + exclude = {} + clean_me = {} + pkgs = {} + saved = {} + deprecated = {} + installed_included = False + # create a big CPV->SRC_URI dict of packages + # whose distfiles should be kept + if (not destructive) or fetch_restricted: + self.output("...non-destructive type search") + pkgs, _deprecated = self._non_destructive(destructive, fetch_restricted) + deprecated.update(_deprecated) + installed_included = True + if destructive: + self.output( + "...destructive type search: %d packages already found" % len(pkgs) + ) + pkgs, _deprecated = self._destructive( + package_names, exclude, pkgs, installed_included + ) + deprecated.update(_deprecated) + # gather the files to be cleaned + self.output("...checking limits for %d ebuild sources" % len(pkgs)) + + checks = self._get_default_checks(size_limit, time_limit, exclude, destructive) + checks.extend(extra_checks) + clean_me = self._check_limits(_distdir, checks, clean_me) + # remove any protected files from the list + self.output( + "...removing protected sources from %s candidates to clean" % len(clean_me) + ) + clean_me = self._remove_protected(pkgs, clean_me) + if not deprecate and len(exclude) and len(clean_me): + self.output( + "...checking final for exclusion from " + + "%s remaining candidates to clean" % len(clean_me) + ) + clean_me, saved = self._check_excludes(exclude, clean_me) + return clean_me, saved, deprecated + + # begin _check_limits code block + + def _get_default_checks(self, size_limit, time_limit, excludes, destructive): + # checks =[(self._isreg_check_, "is_reg_check")] + checks = [self._isreg_check_] + if "filenames" in excludes: + # checks.append((partial(self._filenames_check_, excludes), "Filenames_check")) + checks.append(partial(self._filenames_check_, excludes)) + else: + self.output(" - skipping exclude filenames check") + if size_limit: + # checks.append((partial(self._size_check_, size_limit), "size_check")) + checks.append(partial(self._size_check_, size_limit)) + else: + self.output(" - skipping size limit check") + if time_limit: + # print("time_limit = ", time_limit/1000000,"M sec") + # checks.append((partial(self._time_check_, time_limit), "time_check")) + checks.append(partial(self._time_check_, time_limit)) + else: + self.output(" - skipping time limit check") + if destructive: + self.output(" - skipping dot files check") + else: + checks.append(self._dotfile_check_) + return checks + + def _check_limits(self, _distdir, checks, clean_me=None): + """Checks files if they exceed size and/or time_limits, etc. + + To start with everything is considered dirty and is excluded + only if it matches some condition. + """ + if clean_me is None: + clean_me = {} + for file in os.listdir(_distdir): + filepath = os.path.join(_distdir, file) + try: + file_stat = os.lstat(filepath) + except EnvironmentError: + continue + is_dirty = False + # for check, check_name in checks: + for check in checks: + should_break, is_dirty = check(file_stat, file) + if should_break: + break + + if is_dirty: + # print( "%s Adding file to clean_list:" %check_name, file) + clean_me[file] = [filepath] + return clean_me + + @staticmethod + def _isreg_check_(file_stat, file): + """check if file is a regular file.""" + is_reg_file = stat.S_ISREG(file_stat[stat.ST_MODE]) + return not is_reg_file, is_reg_file + + @staticmethod + def _size_check_(size_limit, file_stat, file): + """checks if the file size exceeds the size_limit""" + if file_stat[stat.ST_SIZE] >= size_limit: + # print( "size mismatch ", file, file_stat[stat.ST_SIZE]) + return True, False + return False, True + + @staticmethod + def _time_check_(time_limit, file_stat, file): + """checks if the file exceeds the time_limit + (think forward, not back, time keeps increasing)""" + if file_stat[stat.ST_MTIME] >= time_limit: + # print( "time match too young ", file, file_stat[stat.ST_MTIME]/1000000,"M sec.") + return True, False + # print( "time match too old", file, file_stat[stat.ST_MTIME]/1000000,"M sec.") + return False, True + + @staticmethod + def _filenames_check_(exclude, file_stat, file): + """checks if the file matches an exclusion file listing""" + # Try to match file name directly + if file in exclude["filenames"]: + return True, False + # See if file matches via regular expression matching + else: + file_match = False + for file_entry in exclude["filenames"]: + if exclude["filenames"][file_entry].match(file): + file_match = True + break + if file_match: + # print( "filename match ", file) + return True, False + return False, True + + @staticmethod + def _dotfile_check_(file_stat, file): + """check if file is a regular file.""" + head, tail = os.path.split(file) + if tail: + is_dot_file = tail.startswith(".") + return is_dot_file, not is_dot_file + + # end _check_limits code block + + @staticmethod + def _remove_protected(pkgs, clean_me): + """Remove files owned by some protected packages. + + @returns packages to clean + @rtype: dictionary + """ + for cpv in pkgs: + uris = pkgs[cpv].split() + uris.reverse() + while uris: + uri = uris.pop() + if uris and uris[-1] == "->": + operator = uris.pop() # noqa + file = uris.pop() + else: + file = os.path.basename(uri) + if file in clean_me: + del clean_me[file] + # no need to waste IO time if there is nothing left to clean + if not len(clean_me): + return clean_me + return clean_me + + def _non_destructive( + self, destructive, fetch_restricted, pkgs_=None, hosts_cpvs=None + ): + """performs the non-destructive checks + + @param destructive: boolean + @param pkgs_: starting dictionary to add to + defaults to {}. + + @returns packages and thier SRC_URI's: {cpv: src_uri,} + @rtype: dictionary + """ + if pkgs_ is None: + pkgs = {} + else: + pkgs = pkgs_.copy() + deprecated = {} + # the following code block was split to optimize for speed + # list all CPV from portree (yeah, that takes time...) + self.output(" - getting complete ebuild list") + cpvs = set(self.portdb.cpv_all()) + installed_cpvs = set(self.vardb.cpv_all()) + # now add any installed cpv's that are not in the tree or overlays + cpvs.update(installed_cpvs) + # Add any installed cpvs from hosts on the network, if any + if hosts_cpvs: + cpvs.update(hosts_cpvs) + installed_cpvs.update(hosts_cpvs) + if fetch_restricted and destructive: + self.output( + " - getting source file names " + + "for %d installed ebuilds" % len(installed_cpvs) + ) + pkgs, _deprecated = self._unrestricted(pkgs, installed_cpvs) + deprecated.update(_deprecated) + # remove the installed cpvs then check the remaining for fetch restiction + cpvs.difference_update(installed_cpvs) + self.output( + " - getting fetch-restricted source file names " + + "for %d remaining ebuilds" % len(cpvs) + ) + pkgs, _deprecated = self._fetch_restricted(pkgs, cpvs) + deprecated.update(_deprecated) + # save the installed cpv list to re-use in _destructive() + self.installed_cpvs = installed_cpvs.copy() + else: + self.output( + " - getting source file names " + "for %d ebuilds" % len(cpvs) + ) + pkgs, _deprecated = self._unrestricted(pkgs, cpvs) + deprecated.update(_deprecated) + return pkgs, deprecated + + def _fetch_restricted(self, pkgs_, cpvs): + """perform fetch restricted non-destructive source + filename lookups + + @param pkgs_: starting dictionary to add to + @param cpvs: set of (cat/pkg-ver, ...) identifiers + + @return a new pkg dictionary + @rtype: dictionary + """ + if pkgs_ is None: + pkgs = {} + else: + pkgs = pkgs_.copy() + deprecated = {} + for cpv in cpvs: + # get SRC_URI and RESTRICT from aux_get + try: # main portdb + (src_uri, restrict) = self.portdb.aux_get(cpv, ["SRC_URI", "RESTRICT"]) + # keep fetch-restricted check + # inside try so it is bypassed on KeyError + if "fetch" in restrict: + pkgs[cpv] = src_uri + except KeyError: + try: # installed vardb + (src_uri, restrict) = self.vardb.aux_get( + cpv, ["SRC_URI", "RESTRICT"] + ) + deprecated[cpv] = src_uri + self.output(DEPRECATED % cpv) + # keep fetch-restricted check + # inside try so it is bypassed on KeyError + if "fetch" in restrict: + pkgs[cpv] = src_uri + except KeyError: + self.output(" - Key Error looking up: " + cpv) + return pkgs, deprecated + + def _unrestricted(self, pkgs_, cpvs): + """Perform unrestricted source filenames lookups + + @param pkgs_: starting packages dictionary + @param cpvs: set of (cat/pkg-ver, ...) identifiers + + @return a new pkg dictionary + @rtype: dictionary + """ + if pkgs_ is None: + pkgs = {} + else: + pkgs = pkgs_.copy() + deprecated = {} + for cpv in cpvs: + # get SRC_URI from aux_get + try: + pkgs[cpv] = self.portdb.aux_get(cpv, ["SRC_URI"])[0] + except KeyError: + try: # installed vardb + pkgs[cpv] = self.vardb.aux_get(cpv, ["SRC_URI"])[0] + deprecated[cpv] = pkgs[cpv] + self.output(DEPRECATED % cpv) + except KeyError: + self.output(" - Key Error looking up: " + cpv) + return pkgs, deprecated + + def _destructive( + self, package_names, exclude, pkgs_=None, installed_included=False + ): + """Builds on pkgs according to input options + + @param package_names: boolean + @param exclude: an exclusion dict as defined in + exclude.parseExcludeFile class. + @param pkgs: starting dictionary to add to + defaults to {}. + @param installed_included: bool. pkgs already + has the installed cpv's added. + + @returns pkgs: {cpv: src_uri,} + """ + if pkgs_ is None: + pkgs = {} + else: + pkgs = pkgs_.copy() + deprecated = {} + pkgset = set() + if not installed_included: + if not package_names: + # list all installed CPV's from vartree + # print( "_destructive: getting vardb.cpv_all") + if not self.installed_cpvs: + pkgset.update(self.vardb.cpv_all()) + else: + pkgset.update(self.installed_cpvs) + self.output(" - processing %s installed ebuilds" % len(pkgset)) + elif package_names: + # list all CPV's from portree for CP's in vartree + # print( "_destructive: getting vardb.cp_all") + cps = self.vardb.cp_all() + self.output(" - processing %s installed packages" % len(cps)) + for package in cps: + pkgset.update(self.portdb.cp_list(package)) + self.output(" - processing excluded") + excludes = self._get_excludes(exclude) + excludes_length = len(excludes) + dprint("excludes", "EXCLUDES LENGTH =%d" % excludes_length) + pkgset.update(excludes) + pkgs_done = set(list(pkgs)) + pkgset.difference_update(pkgs_done) + self.output( + " - (%d of %d total) additional excluded packages to get source filenames for" + % (len(pkgset), excludes_length) + ) + # self.output(" - processing %d ebuilds for filenames" %len(pkgset)) + pkgs, _deprecated = self._unrestricted(pkgs, pkgset) + deprecated.update(_deprecated) + # self.output(" - done...") + return pkgs, deprecated + + def _get_excludes(self, exclude): + """Expands the exclude dictionary into a set of + CPV's + + @param exclude: dictionary of exclusion categories, + packages to exclude from the cleaning + + @rtype: set + @return set of package cpv's + """ + pkgset = set() + for cp in exclDictExpand(exclude): + # add packages from the exclude file + dprint( + "excludes", + "_GET_EXCLUDES, cp=" + cp + ", " + str(self.portdb.cp_list(cp)), + ) + pkgset.update(self.portdb.cp_list(cp)) + return pkgset + + def _check_excludes(self, exclude, clean_me): + """Performs a last minute check on remaining filenames + to see if they should be protected. Since if the pkg-version + was deprecated it would not have been matched to a + source filename and removed. + + @param exclude: an exclusion dictionary + @param clean_me: the list of filenames for cleaning + + @rtype: dict of packages to clean + """ + saved = {} + pn_excludes = exclDictExpandPkgname(exclude) + dprint("excludes", "_check_excludes: made it here ;)") + if not pn_excludes: + return clean_me, saved + dprint("excludes", pn_excludes) + for key in list(clean_me): + if exclMatchFilename(pn_excludes, key): + saved[key] = clean_me[key] + del clean_me[key] + self.output(" ...Saved excluded package filename: " + key) + return clean_me, saved def _deps_equal(deps_a, eapi_a, deps_b, eapi_b, uselist=None): - """Compare two dependency lists given a set of USE flags""" - if deps_a == deps_b: return True + """Compare two dependency lists given a set of USE flags""" + if deps_a == deps_b: + return True - deps_a = use_reduce(deps_a, uselist=uselist, eapi=eapi_a, token_class=Atom) - deps_b = use_reduce(deps_b, uselist=uselist, eapi=eapi_b, token_class=Atom) - strip_slots(deps_a) - strip_slots(deps_b) - return deps_a == deps_b + deps_a = use_reduce(deps_a, uselist=uselist, eapi=eapi_a, token_class=Atom) + deps_b = use_reduce(deps_b, uselist=uselist, eapi=eapi_b, token_class=Atom) + strip_slots(deps_a) + strip_slots(deps_b) + return deps_a == deps_b def findPackages( - options, - exclude=None, - destructive=False, - time_limit=0, - package_names=False, - pkgdir=None, - port_dbapi=portage.db[portage.root]["porttree"].dbapi, - var_dbapi=portage.db[portage.root]["vartree"].dbapi - ): - """Find obsolete binary packages. - - @param options: dict of options determined at runtime - @type options: dict - @param exclude: exclusion dict (as defined in the exclude.parseExcludeFile class) - @type exclude: dict, optional - @param destructive: binpkg is obsolete if not installed (default: `False`) - @type destructive: bool, optional - @param time_limit: exclude binpkg if newer than time value as returned by parseTime() - @type time_limit: int, optional - @param package_names: exclude all binpkg versions if package is installed - (used with `destructive=True`) (default: `False`) - @type package_names: bool, optional - @param pkgdir: path to the binpkg cache (PKGDIR) - @type pkgdir: str - @param port_dbapi: defaults to portage.db[portage.root]["porttree"].dbapi - Can be overridden for tests. - @param var_dbapi: defaults to portage.db[portage.root]["vartree"].dbapi - Can be overridden for tests. - - @return binary packages to remove. e.g. {'cat/pkg-ver': [filepath]} - @rtype: dict - """ - if exclude is None: - exclude = {} - - # Access test, os.walk does not error for "no read permission" - try: - test = os.listdir(pkgdir) - del test - except EnvironmentError as er: - if options['ignore-failure']: - exit(0) - print(pp.error("Error accessing PKGDIR."), file=sys.stderr) - print(pp.error("(Check your make.conf file and environment)."), file=sys.stderr) - print(pp.error("Error: %s" % str(er)), file=sys.stderr) - exit(1) - - # Create a dictionary of all installed packages - if destructive and package_names: - installed = dict.fromkeys(var_dbapi.cp_all()) - else: - installed = {} - - # Dictionary of binary packages to clean. Organized as cpv->[pkgs] in order - # to support FEATURES=binpkg-multi-instance. - dead_binpkgs = {} - - bin_dbapi = portage.binarytree(pkgdir=pkgdir, settings=var_dbapi.settings).dbapi - for cpv in bin_dbapi.cpv_all(): - cp = portage.cpv_getkey(cpv) - - # Exclude per --exclude-file=... - if exclDictMatchCP(exclude, cp): - continue - - # Exclude if binpkg is newer than --time-limit=... - if time_limit: - mtime = int(bin_dbapi.aux_get(cpv, ['_mtime_'])[0]) - if mtime >= time_limit: - continue - - # Exclude if binpkg exists in the porttree and not --deep - if not destructive and port_dbapi.cpv_exists(cpv): - if not options['changed-deps']: - continue - - dep_keys = ('RDEPEND', 'PDEPEND') - keys = ('EAPI', 'USE') + dep_keys - binpkg_metadata = dict(zip(keys, bin_dbapi.aux_get(cpv, keys))) - ebuild_metadata = dict(zip(keys, port_dbapi.aux_get(cpv, keys))) - - if _deps_equal(' '.join(binpkg_metadata[key] for key in dep_keys), binpkg_metadata['EAPI'], - ' '.join(ebuild_metadata[key] for key in dep_keys), ebuild_metadata['EAPI'], - frozenset(binpkg_metadata['USE'].split())): - continue - - if destructive and var_dbapi.cpv_exists(cpv): - # Exclude if an instance of the package is installed due to - # the --package-names option. - if cp in installed and port_dbapi.cpv_exists(cpv): - continue - - # Exclude if BUILD_TIME of binpkg is same as vartree - buildtime = var_dbapi.aux_get(cpv, ['BUILD_TIME'])[0] - if buildtime == bin_dbapi.aux_get(cpv, ['BUILD_TIME'])[0]: - continue - - binpkg_path = bin_dbapi.bintree.getname(cpv) - dead_binpkgs.setdefault(cpv, []).append(binpkg_path) - - return dead_binpkgs + options, + exclude=None, + destructive=False, + time_limit=0, + package_names=False, + pkgdir=None, + port_dbapi=portage.db[portage.root]["porttree"].dbapi, + var_dbapi=portage.db[portage.root]["vartree"].dbapi, +): + """Find obsolete binary packages. + + @param options: dict of options determined at runtime + @type options: dict + @param exclude: exclusion dict (as defined in the exclude.parseExcludeFile class) + @type exclude: dict, optional + @param destructive: binpkg is obsolete if not installed (default: `False`) + @type destructive: bool, optional + @param time_limit: exclude binpkg if newer than time value as returned by parseTime() + @type time_limit: int, optional + @param package_names: exclude all binpkg versions if package is installed + (used with `destructive=True`) (default: `False`) + @type package_names: bool, optional + @param pkgdir: path to the binpkg cache (PKGDIR) + @type pkgdir: str + @param port_dbapi: defaults to portage.db[portage.root]["porttree"].dbapi + Can be overridden for tests. + @param var_dbapi: defaults to portage.db[portage.root]["vartree"].dbapi + Can be overridden for tests. + + @return binary packages to remove. e.g. {'cat/pkg-ver': [filepath]} + @rtype: dict + """ + if exclude is None: + exclude = {} + + # Access test, os.walk does not error for "no read permission" + try: + test = os.listdir(pkgdir) + del test + except EnvironmentError as er: + if options["ignore-failure"]: + exit(0) + print(pp.error("Error accessing PKGDIR."), file=sys.stderr) + print(pp.error("(Check your make.conf file and environment)."), file=sys.stderr) + print(pp.error("Error: %s" % str(er)), file=sys.stderr) + exit(1) + + # Create a dictionary of all installed packages + if destructive and package_names: + installed = dict.fromkeys(var_dbapi.cp_all()) + else: + installed = {} + + # Dictionary of binary packages to clean. Organized as cpv->[pkgs] in order + # to support FEATURES=binpkg-multi-instance. + dead_binpkgs = {} + + bin_dbapi = portage.binarytree(pkgdir=pkgdir, settings=var_dbapi.settings).dbapi + for cpv in bin_dbapi.cpv_all(): + cp = portage.cpv_getkey(cpv) + + # Exclude per --exclude-file=... + if exclDictMatchCP(exclude, cp): + continue + + # Exclude if binpkg is newer than --time-limit=... + if time_limit: + mtime = int(bin_dbapi.aux_get(cpv, ["_mtime_"])[0]) + if mtime >= time_limit: + continue + + # Exclude if binpkg exists in the porttree and not --deep + if not destructive and port_dbapi.cpv_exists(cpv): + if not options["changed-deps"]: + continue + + dep_keys = ("RDEPEND", "PDEPEND") + keys = ("EAPI", "USE") + dep_keys + binpkg_metadata = dict(zip(keys, bin_dbapi.aux_get(cpv, keys))) + ebuild_metadata = dict(zip(keys, port_dbapi.aux_get(cpv, keys))) + + if _deps_equal( + " ".join(binpkg_metadata[key] for key in dep_keys), + binpkg_metadata["EAPI"], + " ".join(ebuild_metadata[key] for key in dep_keys), + ebuild_metadata["EAPI"], + frozenset(binpkg_metadata["USE"].split()), + ): + continue + + if destructive and var_dbapi.cpv_exists(cpv): + # Exclude if an instance of the package is installed due to + # the --package-names option. + if cp in installed and port_dbapi.cpv_exists(cpv): + continue + + # Exclude if BUILD_TIME of binpkg is same as vartree + buildtime = var_dbapi.aux_get(cpv, ["BUILD_TIME"])[0] + if buildtime == bin_dbapi.aux_get(cpv, ["BUILD_TIME"])[0]: + continue + + binpkg_path = bin_dbapi.bintree.getname(cpv) + dead_binpkgs.setdefault(cpv, []).append(binpkg_path) + + return dead_binpkgs + # vim: set ts=4 sw=4 tw=79: |