diff options
Diffstat (limited to 'repoman/pym/repoman/modules/scan')
34 files changed, 3675 insertions, 0 deletions
diff --git a/repoman/pym/repoman/modules/scan/__init__.py b/repoman/pym/repoman/modules/scan/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/repoman/pym/repoman/modules/scan/__init__.py diff --git a/repoman/pym/repoman/modules/scan/depend/__init__.py b/repoman/pym/repoman/modules/scan/depend/__init__.py new file mode 100644 index 000000000..6d1228601 --- /dev/null +++ b/repoman/pym/repoman/modules/scan/depend/__init__.py @@ -0,0 +1,32 @@ +# Copyright 2015-2016 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +doc = """Depend plug-in module for repoman. +Performs Dependency checks on ebuilds.""" +__doc__ = doc[:] + + +module_spec = { + 'name': 'depend', + 'description': doc, + 'provides':{ + 'profile-module': { + 'name': "profile", + 'sourcefile': "profile", + 'class': "ProfileDependsChecks", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + 'mod_kwargs': ['qatracker', 'portdb', 'profiles', 'options', + 'repo_metadata', 'repo_settings', 'include_arches', 'caches', + 'repoman_incrementals', 'env', 'have', 'dev_keywords' + ], + 'func_kwargs': { + 'ebuild': (None, None), + 'pkg': (None, None), + }, + }, + } +} + diff --git a/repoman/pym/repoman/modules/scan/depend/_depend_checks.py b/repoman/pym/repoman/modules/scan/depend/_depend_checks.py new file mode 100644 index 000000000..4e1d216e1 --- /dev/null +++ b/repoman/pym/repoman/modules/scan/depend/_depend_checks.py @@ -0,0 +1,150 @@ +# -*- coding:utf-8 -*- + + +from _emerge.Package import Package + +from repoman.check_missingslot import check_missingslot +# import our initialized portage instance +from repoman._portage import portage +from repoman.qa_data import suspect_virtual, suspect_rdepend + + +def _depend_checks(ebuild, pkg, portdb, qatracker, repo_metadata): + '''Checks the ebuild dependencies for errors + + @param pkg: Package in which we check (object). + @param ebuild: Ebuild which we check (object). + @param portdb: portdb instance + @param qatracker: QATracker instance + @param repo_metadata: dictionary of various repository items. + @returns: (unknown_pkgs, badlicsyntax) + ''' + + unknown_pkgs = set() + + inherited_java_eclass = "java-pkg-2" in ebuild.inherited or \ + "java-pkg-opt-2" in ebuild.inherited, + inherited_wxwidgets_eclass = "wxwidgets" in ebuild.inherited + # operator_tokens = set(["||", "(", ")"]) + type_list, badsyntax = [], [] + for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES", "PROVIDE"): + mydepstr = ebuild.metadata[mytype] + + buildtime = mytype in Package._buildtime_keys + runtime = mytype in Package._runtime_keys + token_class = None + if mytype.endswith("DEPEND"): + token_class = portage.dep.Atom + + try: + atoms = portage.dep.use_reduce( + mydepstr, matchall=1, flat=True, + is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class) + except portage.exception.InvalidDependString as e: + atoms = None + badsyntax.append(str(e)) + + if atoms and mytype.endswith("DEPEND"): + if runtime and \ + "test?" in mydepstr.split(): + qatracker.add_error( + mytype + '.suspect', + "%s: 'test?' USE conditional in %s" % + (ebuild.relative_path, mytype)) + + for atom in atoms: + if atom == "||": + continue + + is_blocker = atom.blocker + + # Skip dependency.unknown for blockers, so that we + # don't encourage people to remove necessary blockers, + # as discussed in bug 382407. We use atom.without_use + # due to bug 525376. + if not is_blocker and \ + not portdb.xmatch("match-all", atom.without_use) and \ + not atom.cp.startswith("virtual/"): + unknown_pkgs.add((mytype, atom.unevaluated_atom)) + + if pkg.category != "virtual": + if not is_blocker and \ + atom.cp in suspect_virtual: + qatracker.add_error( + 'virtual.suspect', ebuild.relative_path + + ": %s: consider using '%s' instead of '%s'" % + (mytype, suspect_virtual[atom.cp], atom)) + if not is_blocker and \ + atom.cp.startswith("perl-core/"): + qatracker.add_error('dependency.perlcore', + ebuild.relative_path + + ": %s: please use '%s' instead of '%s'" % + (mytype, + atom.replace("perl-core/","virtual/perl-"), + atom)) + + if buildtime and \ + not is_blocker and \ + not inherited_java_eclass and \ + atom.cp == "virtual/jdk": + qatracker.add_error( + 'java.eclassesnotused', ebuild.relative_path) + elif buildtime and \ + not is_blocker and \ + not inherited_wxwidgets_eclass and \ + atom.cp == "x11-libs/wxGTK": + qatracker.add_error( + 'wxwidgets.eclassnotused', + "%s: %ss on x11-libs/wxGTK without inheriting" + " wxwidgets.eclass" % (ebuild.relative_path, mytype)) + elif runtime: + if not is_blocker and \ + atom.cp in suspect_rdepend: + qatracker.add_error( + mytype + '.suspect', + ebuild.relative_path + ": '%s'" % atom) + + if atom.operator == "~" and \ + portage.versions.catpkgsplit(atom.cpv)[3] != "r0": + qacat = 'dependency.badtilde' + qatracker.add_error( + qacat, "%s: %s uses the ~ operator" + " with a non-zero revision: '%s'" % + (ebuild.relative_path, mytype, atom)) + + check_missingslot(atom, mytype, ebuild.eapi, portdb, qatracker, + ebuild.relative_path, ebuild.metadata) + + type_list.extend([mytype] * (len(badsyntax) - len(type_list))) + + for m, b in zip(type_list, badsyntax): + if m.endswith("DEPEND"): + qacat = "dependency.syntax" + else: + qacat = m + ".syntax" + qatracker.add_error( + qacat, "%s: %s: %s" % (ebuild.relative_path, m, b)) + + # data required for some other tests + badlicsyntax = len([z for z in type_list if z == "LICENSE"]) + badprovsyntax = len([z for z in type_list if z == "PROVIDE"]) + baddepsyntax = len(type_list) != badlicsyntax + badprovsyntax + badlicsyntax = badlicsyntax > 0 + #badprovsyntax = badprovsyntax > 0 + + # Parse the LICENSE variable, remove USE conditions and flatten it. + licenses = portage.dep.use_reduce( + ebuild.metadata["LICENSE"], matchall=1, flat=True) + + # Check each entry to ensure that it exists in ${PORTDIR}/licenses/. + for lic in licenses: + # Need to check for "||" manually as no portage + # function will remove it without removing values. + if lic not in repo_metadata['liclist'] and lic != "||": + qatracker.add_error("LICENSE.invalid", + "%s: %s" % (ebuild.relative_path, lic)) + elif lic in repo_metadata['lic_deprecated']: + qatracker.add_error("LICENSE.deprecated", + "%s: %s" % (ebuild.relative_path, lic)) + + return unknown_pkgs, baddepsyntax diff --git a/repoman/pym/repoman/modules/scan/depend/_gen_arches.py b/repoman/pym/repoman/modules/scan/depend/_gen_arches.py new file mode 100644 index 000000000..16b8dac5f --- /dev/null +++ b/repoman/pym/repoman/modules/scan/depend/_gen_arches.py @@ -0,0 +1,57 @@ +# -*- coding:utf-8 -*- + + +def _gen_arches(ebuild, options, repo_settings, profiles): + '''Determines the arches for the ebuild following the profile rules + + @param ebuild: Ebuild which we check (object). + @param profiles: dictionary + @param options: cli options + @param repo_settings: repository settings instance + @returns: dictionary, including arches set + ''' + if options.ignore_arches: + arches = [[ + repo_settings.repoman_settings["ARCH"], repo_settings.repoman_settings["ARCH"], + repo_settings.repoman_settings["ACCEPT_KEYWORDS"].split()]] + else: + arches = set() + for keyword in ebuild.keywords: + if keyword[0] == "-": + continue + elif keyword[0] == "~": + arch = keyword[1:] + if arch == "*": + for expanded_arch in profiles: + if expanded_arch == "**": + continue + arches.add( + (keyword, expanded_arch, ( + expanded_arch, "~" + expanded_arch))) + else: + arches.add((keyword, arch, (arch, keyword))) + else: + # For ebuilds with stable keywords, check if the + # dependencies are satisfiable for unstable + # configurations, since use.stable.mask is not + # applied for unstable configurations (see bug + # 563546). + if keyword == "*": + for expanded_arch in profiles: + if expanded_arch == "**": + continue + arches.add( + (keyword, expanded_arch, (expanded_arch,))) + arches.add( + (keyword, expanded_arch, + (expanded_arch, "~" + expanded_arch))) + else: + arches.add((keyword, keyword, (keyword,))) + arches.add((keyword, keyword, + (keyword, "~" + keyword))) + if not arches: + # Use an empty profile for checking dependencies of + # packages that have empty KEYWORDS. + arches.add(('**', '**', ('**',))) + + return arches diff --git a/repoman/pym/repoman/modules/scan/depend/profile.py b/repoman/pym/repoman/modules/scan/depend/profile.py new file mode 100644 index 000000000..a714a9317 --- /dev/null +++ b/repoman/pym/repoman/modules/scan/depend/profile.py @@ -0,0 +1,256 @@ +# -*- coding:utf-8 -*- + + +import copy +from pprint import pformat + +from _emerge.Package import Package + +# import our initialized portage instance +from repoman._portage import portage +from repoman.modules.scan.scanbase import ScanBase +from repoman.modules.scan.depend._depend_checks import _depend_checks +from repoman.modules.scan.depend._gen_arches import _gen_arches +from portage.dep import Atom + + +def sort_key(item): + return item[2].sub_path + + +class ProfileDependsChecks(ScanBase): + '''Perform dependency checks for the different profiles''' + + def __init__(self, **kwargs): + '''Class init + + @param qatracker: QATracker instance + @param portdb: portdb instance + @param profiles: dictionary + @param options: cli options + @param repo_settings: repository settings instance + @param include_arches: set + @param caches: dictionary of our caches + @param repoman_incrementals: tuple + @param env: the environment + @param have: dictionary instance + @param dev_keywords: developer profile keywords + @param repo_metadata: dictionary of various repository items. + ''' + self.qatracker = kwargs.get('qatracker') + self.portdb = kwargs.get('portdb') + self.profiles = kwargs.get('profiles') + self.options = kwargs.get('options') + self.repo_settings = kwargs.get('repo_settings') + self.include_arches = kwargs.get('include_arches') + self.caches = kwargs.get('caches') + self.repoman_incrementals = kwargs.get('repoman_incrementals') + self.env = kwargs.get('env') + self.have = kwargs.get('have') + self.dev_keywords = kwargs.get('dev_keywords') + self.repo_metadata = kwargs.get('repo_metadata') + + def check(self, **kwargs): + '''Perform profile dependant dependancy checks + + @param arches: + @param pkg: Package in which we check (object). + @param ebuild: Ebuild which we check (object). + @param baddepsyntax: boolean + @param unknown_pkgs: set of tuples (type, atom.unevaluated_atom) + @returns: dictionary + ''' + ebuild = kwargs.get('ebuild').get() + pkg = kwargs.get('pkg').get() + unknown_pkgs, baddepsyntax = _depend_checks( + ebuild, pkg, self.portdb, self.qatracker, self.repo_metadata) + + relevant_profiles = [] + for keyword, arch, groups in _gen_arches(ebuild, self.options, + self.repo_settings, self.profiles): + if arch not in self.profiles: + # A missing profile will create an error further down + # during the KEYWORDS verification. + continue + + if self.include_arches is not None: + if arch not in self.include_arches: + continue + + relevant_profiles.extend( + (keyword, groups, prof) for prof in self.profiles[arch]) + + relevant_profiles.sort(key=sort_key) + + for keyword, groups, prof in relevant_profiles: + + is_stable_profile = prof.status == "stable" + is_dev_profile = prof.status == "dev" and \ + self.options.include_dev + is_exp_profile = prof.status == "exp" and \ + self.options.include_exp_profiles == 'y' + if not (is_stable_profile or is_dev_profile or is_exp_profile): + continue + + dep_settings = self.caches['arch'].get(prof.sub_path) + if dep_settings is None: + dep_settings = portage.config( + config_profile_path=prof.abs_path, + config_incrementals=self.repoman_incrementals, + config_root=self.repo_settings.config_root, + local_config=False, + _unmatched_removal=self.options.unmatched_removal, + env=self.env, repositories=self.repo_settings.repoman_settings.repositories) + dep_settings.categories = self.repo_settings.repoman_settings.categories + if self.options.without_mask: + dep_settings._mask_manager_obj = \ + copy.deepcopy(dep_settings._mask_manager) + dep_settings._mask_manager._pmaskdict.clear() + self.caches['arch'][prof.sub_path] = dep_settings + + xmatch_cache_key = (prof.sub_path, tuple(groups)) + xcache = self.caches['arch_xmatch'].get(xmatch_cache_key) + if xcache is None: + self.portdb.melt() + self.portdb.freeze() + xcache = self.portdb.xcache + xcache.update(self.caches['shared_xmatch']) + self.caches['arch_xmatch'][xmatch_cache_key] = xcache + + self.repo_settings.trees[self.repo_settings.root]["porttree"].settings = dep_settings + self.portdb.settings = dep_settings + self.portdb.xcache = xcache + + dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups) + # just in case, prevent config.reset() from nuking these. + dep_settings.backup_changes("ACCEPT_KEYWORDS") + + # This attribute is used in dbapi._match_use() to apply + # use.stable.{mask,force} settings based on the stable + # status of the parent package. This is required in order + # for USE deps of unstable packages to be resolved correctly, + # since otherwise use.stable.{mask,force} settings of + # dependencies may conflict (see bug #456342). + dep_settings._parent_stable = dep_settings._isStable(pkg) + + # Handle package.use*.{force,mask) calculation, for use + # in dep_check. + dep_settings.useforce = dep_settings._use_manager.getUseForce( + pkg, stable=dep_settings._parent_stable) + dep_settings.usemask = dep_settings._use_manager.getUseMask( + pkg, stable=dep_settings._parent_stable) + + if not baddepsyntax: + ismasked = not ebuild.archs or \ + pkg.cpv not in self.portdb.xmatch("match-visible", + Atom("%s::%s" % (pkg.cp, self.repo_settings.repo_config.name))) + if ismasked: + if not self.have['pmasked']: + self.have['pmasked'] = bool(dep_settings._getMaskAtom( + pkg.cpv, ebuild.metadata)) + if self.options.ignore_masked: + continue + # we are testing deps for a masked package; give it some lee-way + suffix = "masked" + matchmode = "minimum-all-ignore-profile" + else: + suffix = "" + matchmode = "minimum-visible" + + if not self.have['dev_keywords']: + self.have['dev_keywords'] = \ + bool(self.dev_keywords.intersection(ebuild.keywords)) + + if prof.status == "dev": + suffix = suffix + "indev" + + for mytype in Package._dep_keys: + + mykey = "dependency.bad" + suffix + myvalue = ebuild.metadata[mytype] + if not myvalue: + continue + + success, atoms = portage.dep_check( + myvalue, self.portdb, dep_settings, + use="all", mode=matchmode, trees=self.repo_settings.trees) + + if success: + if atoms: + + # Don't bother with dependency.unknown for + # cases in which *DEPEND.bad is triggered. + for atom in atoms: + # dep_check returns all blockers and they + # aren't counted for *DEPEND.bad, so we + # ignore them here. + if not atom.blocker: + unknown_pkgs.discard( + (mytype, atom.unevaluated_atom)) + + if not prof.sub_path: + # old-style virtuals currently aren't + # resolvable with empty profile, since + # 'virtuals' mappings are unavailable + # (it would be expensive to search + # for PROVIDE in all ebuilds) + atoms = [ + atom for atom in atoms if not ( + atom.cp.startswith('virtual/') + and not self.portdb.cp_list(atom.cp))] + + # we have some unsolvable deps + # remove ! deps, which always show up as unsatisfiable + all_atoms = [ + str(atom.unevaluated_atom) + for atom in atoms if not atom.blocker] + + # if we emptied out our list, continue: + if not all_atoms: + continue + + # Filter out duplicates. We do this by hand (rather + # than use a set) so the order is stable and better + # matches the order that's in the ebuild itself. + atoms = [] + for atom in all_atoms: + if atom not in atoms: + atoms.append(atom) + + if self.options.output_style in ['column']: + self.qatracker.add_error(mykey, + "%s: %s: %s(%s) %s" + % (ebuild.relative_path, mytype, keyword, + prof, repr(atoms))) + else: + self.qatracker.add_error(mykey, + "%s: %s: %s(%s)\n%s" + % (ebuild.relative_path, mytype, keyword, + prof, pformat(atoms, indent=6))) + else: + if self.options.output_style in ['column']: + self.qatracker.add_error(mykey, + "%s: %s: %s(%s) %s" + % (ebuild.relative_path, mytype, keyword, + prof, repr(atoms))) + else: + self.qatracker.add_error(mykey, + "%s: %s: %s(%s)\n%s" + % (ebuild.relative_path, mytype, keyword, + prof, pformat(atoms, indent=6))) + + if not baddepsyntax and unknown_pkgs: + type_map = {} + for mytype, atom in unknown_pkgs: + type_map.setdefault(mytype, set()).add(atom) + for mytype, atoms in type_map.items(): + self.qatracker.add_error( + "dependency.unknown", "%s: %s: %s" + % (ebuild.relative_path, mytype, ", ".join(sorted(atoms)))) + + return False + + @property + def runInEbuilds(self): + '''Ebuild level scans''' + return (True, [self.check]) diff --git a/repoman/pym/repoman/modules/scan/directories/__init__.py b/repoman/pym/repoman/modules/scan/directories/__init__.py new file mode 100644 index 000000000..47834cb40 --- /dev/null +++ b/repoman/pym/repoman/modules/scan/directories/__init__.py @@ -0,0 +1,48 @@ +# Copyright 2015-2016 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +doc = """Directories plug-in module for repoman. +Performs an FilesChecks check on ebuilds.""" +__doc__ = doc[:] + + +module_spec = { + 'name': 'directories', + 'description': doc, + 'provides':{ + 'directories-module': { + 'name': "files", + 'sourcefile': "files", + 'class': "FileChecks", + 'description': doc, + 'functions': ['check'], + 'func_kwargs': { + }, + 'mod_kwargs': ['portdb', 'qatracker', 'repo_settings', 'vcs_settings', + ], + 'func_kwargs': { + 'changed': (None, None), + 'checkdir': (None, None), + 'checkdirlist': (None, None), + 'checkdir_relative': (None, None), + }, + }, + 'mtime-module': { + 'name': "mtime", + 'sourcefile': "mtime", + 'class': "MtimeChecks", + 'description': doc, + 'functions': ['check'], + 'func_kwargs': { + }, + 'mod_kwargs': ['vcs_settings', + ], + 'func_kwargs': { + 'changed': (None, None), + 'ebuild': (None, None), + 'pkg': (None, None), + }, + }, + } +} + diff --git a/repoman/pym/repoman/modules/scan/directories/files.py b/repoman/pym/repoman/modules/scan/directories/files.py new file mode 100644 index 000000000..2aed26440 --- /dev/null +++ b/repoman/pym/repoman/modules/scan/directories/files.py @@ -0,0 +1,94 @@ +# -*- coding:utf-8 -*- + +'''repoman/checks/diretories/files.py + +''' + +import io + +from portage import _encodings, _unicode_encode +from portage import os + +from repoman.modules.vcs.vcs import vcs_new_changed +from repoman.modules.scan.scanbase import ScanBase + + +class FileChecks(ScanBase): + '''Performs various file checks in the package's directory''' + + def __init__(self, **kwargs): + ''' + @param portdb: portdb instance + @param qatracker: QATracker instance + @param repo_settings: settings instance + @param vcs_settings: VCSSettings instance + ''' + super(FileChecks, self).__init__(**kwargs) + self.portdb = kwargs.get('portdb') + self.qatracker = kwargs.get('qatracker') + self.repo_settings = kwargs.get('repo_settings') + self.repoman_settings = self.repo_settings.repoman_settings + self.vcs_settings = kwargs.get('vcs_settings') + + def check(self, **kwargs): + '''Checks the ebuild sources and files for errors + + @param checkdir: string, directory path + @param checkdir_relative: repolevel determined path + @param changed: dictionary instance + @returns: dictionary + ''' + checkdir = kwargs.get('checkdir') + checkdirlist = kwargs.get('checkdirlist').get() + checkdir_relative = kwargs.get('checkdir_relative') + changed = kwargs.get('changed').changed + new = kwargs.get('changed').new + for y_file in checkdirlist: + index = self.repo_settings.repo_config.find_invalid_path_char(y_file) + if index != -1: + y_relative = os.path.join(checkdir_relative, y_file) + invcs = self.vcs_settings.vcs is not None + inchangeset = vcs_new_changed(y_relative, changed, new) + if invcs and not inchangeset: + # If the file isn't in the VCS new or changed set, then + # assume that it's an irrelevant temporary file (Manifest + # entries are not generated for file names containing + # prohibited characters). See bug #406877. + index = -1 + if index != -1: + self.qatracker.add_error( + "file.name", + "%s/%s: char '%s'" % (checkdir, y_file, y_file[index])) + + if not ( + y_file in ("ChangeLog", "metadata.xml") + or y_file.endswith(".ebuild")): + continue + f = None + try: + line = 1 + f = io.open( + _unicode_encode( + os.path.join(checkdir, y_file), + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content']) + for l in f: + line += 1 + except UnicodeDecodeError as ue: + s = ue.object[:ue.start] + l2 = s.count("\n") + line += l2 + if l2 != 0: + s = s[s.rfind("\n") + 1:] + self.qatracker.add_error( + "file.UTF8", "%s/%s: line %i, just after: '%s'" % ( + checkdir, y_file, line, s)) + finally: + if f is not None: + f.close() + return False + + @property + def runInPkgs(self): + '''Package level scans''' + return (True, [self.check]) diff --git a/repoman/pym/repoman/modules/scan/directories/mtime.py b/repoman/pym/repoman/modules/scan/directories/mtime.py new file mode 100644 index 000000000..134a86b80 --- /dev/null +++ b/repoman/pym/repoman/modules/scan/directories/mtime.py @@ -0,0 +1,30 @@ + +from repoman.modules.scan.scanbase import ScanBase + + +class MtimeChecks(ScanBase): + + def __init__(self, **kwargs): + self.vcs_settings = kwargs.get('vcs_settings') + + def check(self, **kwargs): + '''Perform a changelog and untracked checks on the ebuild + + @param pkg: Package in which we check (object). + @param ebuild: Ebuild which we check (object). + @param changed: dictionary instance + @returns: dictionary + ''' + ebuild = kwargs.get('ebuild').get() + changed = kwargs.get('changed') + pkg = kwargs.get('pkg').get() + if not self.vcs_settings.vcs_preserves_mtime: + if ebuild.ebuild_path not in changed.new_ebuilds and \ + ebuild.ebuild_path not in changed.ebuilds: + pkg.mtime = None + return False + + @property + def runInEbuilds(self): + '''Ebuild level scans''' + return (True, [self.check]) diff --git a/repoman/pym/repoman/modules/scan/eapi/__init__.py b/repoman/pym/repoman/modules/scan/eapi/__init__.py new file mode 100644 index 000000000..4c3dd6e8f --- /dev/null +++ b/repoman/pym/repoman/modules/scan/eapi/__init__.py @@ -0,0 +1,29 @@ +# Copyright 2015-2016 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +doc = """Eapi plug-in module for repoman. +Performs an IsEbuild check on ebuilds.""" +__doc__ = doc[:] + + +module_spec = { + 'name': 'eapi', + 'description': doc, + 'provides':{ + 'live-module': { + 'name': "eapi", + 'sourcefile': "eapi", + 'class': "EAPIChecks", + 'description': doc, + 'functions': ['check'], + 'func_kwargs': { + }, + 'mod_kwargs': ['qatracker', 'repo_settings' + ], + 'func_kwargs': { + 'ebuild': (None, None), + }, + }, + } +} + diff --git a/repoman/pym/repoman/modules/scan/eapi/eapi.py b/repoman/pym/repoman/modules/scan/eapi/eapi.py new file mode 100644 index 000000000..1d4ad5a4a --- /dev/null +++ b/repoman/pym/repoman/modules/scan/eapi/eapi.py @@ -0,0 +1,49 @@ + +'''eapi.py +Perform checks on the EAPI variable. +''' + +from repoman.modules.scan.scanbase import ScanBase + + +class EAPIChecks(ScanBase): + '''Perform checks on the EAPI variable.''' + + def __init__(self, **kwargs): + ''' + @param qatracker: QATracker instance + @param repo_settings: Repository settings + ''' + self.qatracker = kwargs.get('qatracker') + self.repo_settings = kwargs.get('repo_settings') + + def check(self, **kwargs): + ''' + @param pkg: Package in which we check (object). + @param ebuild: Ebuild which we check (object). + @returns: dictionary + ''' + ebuild = kwargs.get('ebuild').get() + + if not self._checkBanned(ebuild): + self._checkDeprecated(ebuild) + return False + + def _checkBanned(self, ebuild): + if self.repo_settings.repo_config.eapi_is_banned(ebuild.eapi): + self.qatracker.add_error( + "repo.eapi.banned", "%s: %s" % (ebuild.relative_path, ebuild.eapi)) + return True + return False + + def _checkDeprecated(self, ebuild): + if self.repo_settings.repo_config.eapi_is_deprecated(ebuild.eapi): + self.qatracker.add_error( + "repo.eapi.deprecated", "%s: %s" % (ebuild.relative_path, ebuild.eapi)) + return True + return False + + @property + def runInEbuilds(self): + '''Ebuild level scans''' + return (True, [self.check]) diff --git a/repoman/pym/repoman/modules/scan/ebuild/__init__.py b/repoman/pym/repoman/modules/scan/ebuild/__init__.py new file mode 100644 index 000000000..8666e78c2 --- /dev/null +++ b/repoman/pym/repoman/modules/scan/ebuild/__init__.py @@ -0,0 +1,58 @@ +# Copyright 2015-2016 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +doc = """Ebuild plug-in module for repoman. +Performs an IsEbuild check on ebuilds.""" +__doc__ = doc[:] + + +module_spec = { + 'name': 'ebuild', + 'description': doc, + 'provides':{ + 'ebuild-module': { + 'name': "ebuild", + 'sourcefile': "ebuild", + 'class': "Ebuild", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + 'mod_kwargs': ['qatracker', 'repo_settings', 'vcs_settings', + 'checks', 'portdb' + ], + 'func_kwargs': { + 'can_force': (None, None), + 'catdir': (None, None), + 'changed': (None, None), + 'changelog_modified': (None, None), + 'checkdir': (None, None), + 'checkdirlist': (None, None), + 'ebuild': ('Future', 'UNSET'), + 'pkg': ('Future', 'UNSET'), + 'pkgdir': (None, None), + 'pkgs': ('Future', 'dict'), + 'repolevel': (None, None), + 'validity_future': (None, None), + 'xpkg': (None, None), + 'y_ebuild': (None, None), + }, + }, + 'multicheck-module': { + 'name': "multicheck", + 'sourcefile': "multicheck", + 'class': "MultiCheck", + 'description': doc, + 'functions': ['check'], + 'func_kwargs': { + }, + 'mod_kwargs': ['qatracker', 'options' + ], + 'func_kwargs': { + 'ebuild': (None, None), + 'pkg': (None, None), + }, + }, + } +} + diff --git a/repoman/pym/repoman/modules/scan/ebuild/checks.py b/repoman/pym/repoman/modules/scan/ebuild/checks.py new file mode 100644 index 000000000..fb3e01944 --- /dev/null +++ b/repoman/pym/repoman/modules/scan/ebuild/checks.py @@ -0,0 +1,1007 @@ +# -*- coding:utf-8 -*- +# repoman: Checks +# Copyright 2007-2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""This module contains functions used in Repoman to ascertain the quality +and correctness of an ebuild.""" + +from __future__ import unicode_literals + +import codecs +from itertools import chain +import re +import time + +# import our initialized portage instance +from repoman._portage import portage + +from portage.eapi import ( + eapi_supports_prefix, eapi_has_implicit_rdepend, + eapi_has_src_prepare_and_src_configure, eapi_has_dosed_dohard, + eapi_exports_AA, eapi_has_pkg_pretend) + +from . import errors + + +class LineCheck(object): + """Run a check on a line of an ebuild.""" + """A regular expression to determine whether to ignore the line""" + ignore_line = False + """True if lines containing nothing more than comments with optional + leading whitespace should be ignored""" + ignore_comment = True + + def new(self, pkg): + pass + + def check_eapi(self, eapi): + """Returns if check should be run in the given EAPI (default: True)""" + return True + + def check(self, num, line): + """Run the check on line and return error if there is one""" + if self.re.match(line): + return self.error + + def end(self): + pass + + +class PhaseCheck(LineCheck): + """ basic class for function detection """ + + func_end_re = re.compile(r'^\}$') + phases_re = re.compile('(%s)' % '|'.join(( + 'pkg_pretend', 'pkg_setup', 'src_unpack', 'src_prepare', + 'src_configure', 'src_compile', 'src_test', 'src_install', + 'pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm', + 'pkg_config'))) + in_phase = '' + + def check(self, num, line): + m = self.phases_re.match(line) + if m is not None: + self.in_phase = m.group(1) + if self.in_phase != '' and self.func_end_re.match(line) is not None: + self.in_phase = '' + + return self.phase_check(num, line) + + def phase_check(self, num, line): + """ override this function for your checks """ + pass + + +class EbuildHeader(LineCheck): + """Ensure ebuilds have proper headers + Copyright header errors + CVS header errors + License header errors + + Args: + modification_year - Year the ebuild was last modified + """ + + repoman_check_name = 'ebuild.badheader' + + gentoo_copyright = r'^# Copyright ((1999|2\d\d\d)-)?%s Gentoo Foundation$' + gentoo_license = ( + '# Distributed under the terms' + ' of the GNU General Public License v2') + id_header = '# $Id$' + ignore_comment = False + + def new(self, pkg): + if pkg.mtime is None: + self.modification_year = r'2\d\d\d' + else: + self.modification_year = str(time.gmtime(pkg.mtime)[0]) + self.gentoo_copyright_re = re.compile( + self.gentoo_copyright % self.modification_year) + + def check(self, num, line): + if num > 2: + return + elif num == 0: + if not self.gentoo_copyright_re.match(line): + return errors.COPYRIGHT_ERROR + elif num == 1 and line.rstrip('\n') != self.gentoo_license: + return errors.LICENSE_ERROR + #elif num == 2 and line.rstrip('\n') != self.id_header: + # return errors.ID_HEADER_ERROR + + +class EbuildWhitespace(LineCheck): + """Ensure ebuilds have proper whitespacing""" + + repoman_check_name = 'ebuild.minorsyn' + + ignore_line = re.compile(r'(^$)|(^(\t)*#)') + ignore_comment = False + leading_spaces = re.compile(r'^[\S\t]') + trailing_whitespace = re.compile(r'.*([\S]$)') + + def check(self, num, line): + if self.leading_spaces.match(line) is None: + return errors.LEADING_SPACES_ERROR + if self.trailing_whitespace.match(line) is None: + return errors.TRAILING_WHITESPACE_ERROR + + +class EbuildBlankLine(LineCheck): + repoman_check_name = 'ebuild.minorsyn' + ignore_comment = False + blank_line = re.compile(r'^$') + + def new(self, pkg): + self.line_is_blank = False + + def check(self, num, line): + if self.line_is_blank and self.blank_line.match(line): + return 'Useless blank line on line: %d' + if self.blank_line.match(line): + self.line_is_blank = True + else: + self.line_is_blank = False + + def end(self): + if self.line_is_blank: + yield 'Useless blank line on last line' + + +class EbuildQuote(LineCheck): + """Ensure ebuilds have valid quoting around things like D,FILESDIR, etc...""" + + repoman_check_name = 'ebuild.minorsyn' + _message_commands = [ + "die", "echo", "eerror", "einfo", "elog", "eqawarn", "ewarn"] + _message_re = re.compile( + r'\s(' + "|".join(_message_commands) + r')\s+"[^"]*"\s*$') + _ignored_commands = ["local", "export"] + _message_commands + ignore_line = re.compile( + r'(^$)|(^\s*#.*)|(^\s*\w+=.*)' + + r'|(^\s*(' + "|".join(_ignored_commands) + r')\s+)') + ignore_comment = False + var_names = ["D", "DISTDIR", "FILESDIR", "S", "T", "ROOT", "WORKDIR"] + + # EAPI=3/Prefix vars + var_names += ["ED", "EPREFIX", "EROOT"] + + # variables for games.eclass + var_names += [ + "Ddir", "GAMES_PREFIX_OPT", "GAMES_DATADIR", + "GAMES_DATADIR_BASE", "GAMES_SYSCONFDIR", "GAMES_STATEDIR", + "GAMES_LOGDIR", "GAMES_BINDIR"] + + # variables for multibuild.eclass + var_names += ["BUILD_DIR"] + + var_names = "(%s)" % "|".join(var_names) + var_reference = re.compile( + r'\$(\{%s\}|%s\W)' % (var_names, var_names)) + missing_quotes = re.compile( + r'(\s|^)[^"\'\s]*\$\{?%s\}?[^"\'\s]*(\s|$)' % var_names) + cond_begin = re.compile(r'(^|\s+)\[\[($|\\$|\s+)') + cond_end = re.compile(r'(^|\s+)\]\]($|\\$|\s+)') + + def check(self, num, line): + if self.var_reference.search(line) is None: + return + # There can be multiple matches / violations on a single line. We + # have to make sure none of the matches are violators. Once we've + # found one violator, any remaining matches on the same line can + # be ignored. + pos = 0 + while pos <= len(line) - 1: + missing_quotes = self.missing_quotes.search(line, pos) + if not missing_quotes: + break + # If the last character of the previous match is a whitespace + # character, that character may be needed for the next + # missing_quotes match, so search overlaps by 1 character. + group = missing_quotes.group() + pos = missing_quotes.end() - 1 + + # Filter out some false positives that can + # get through the missing_quotes regex. + if self.var_reference.search(group) is None: + continue + + # Filter matches that appear to be an + # argument to a message command. + # For example: false || ewarn "foo $WORKDIR/bar baz" + message_match = self._message_re.search(line) + if message_match is not None and \ + message_match.start() < pos and \ + message_match.end() > pos: + break + + # This is an attempt to avoid false positives without getting + # too complex, while possibly allowing some (hopefully + # unlikely) violations to slip through. We just assume + # everything is correct if the there is a ' [[ ' or a ' ]] ' + # anywhere in the whole line (possibly continued over one + # line). + if self.cond_begin.search(line) is not None: + continue + if self.cond_end.search(line) is not None: + continue + + # Any remaining matches on the same line can be ignored. + return errors.MISSING_QUOTES_ERROR + + +class EbuildAssignment(LineCheck): + """Ensure ebuilds don't assign to readonly variables.""" + + repoman_check_name = 'variable.readonly' + read_only_vars = 'A|CATEGORY|P|P[VNRF]|PVR|D|WORKDIR|FILESDIR|FEATURES|USE' + readonly_assignment = re.compile(r'^\s*(export\s+)?(%s)=' % read_only_vars) + + def check(self, num, line): + match = self.readonly_assignment.match(line) + e = None + if match is not None: + e = errors.READONLY_ASSIGNMENT_ERROR + return e + + +class Eapi3EbuildAssignment(EbuildAssignment): + """Ensure ebuilds don't assign to readonly EAPI 3-introduced variables.""" + + readonly_assignment = re.compile(r'\s*(export\s+)?(ED|EPREFIX|EROOT)=') + + def check_eapi(self, eapi): + return eapi_supports_prefix(eapi) + + +class EbuildNestedDie(LineCheck): + """Check ebuild for nested die statements (die statements in subshells)""" + + repoman_check_name = 'ebuild.nesteddie' + nesteddie_re = re.compile(r'^[^#]*\s\(\s[^)]*\bdie\b') + + def check(self, num, line): + if self.nesteddie_re.match(line): + return errors.NESTED_DIE_ERROR + + +class EbuildUselessDodoc(LineCheck): + """Check ebuild for useless files in dodoc arguments.""" + repoman_check_name = 'ebuild.minorsyn' + uselessdodoc_re = re.compile( + r'^\s*dodoc(\s+|\s+.*\s+)(ABOUT-NLS|COPYING|LICENCE|LICENSE)($|\s)') + + def check(self, num, line): + match = self.uselessdodoc_re.match(line) + if match: + return "Useless dodoc '%s'" % (match.group(2), ) + " on line: %d" + + +class EbuildUselessCdS(LineCheck): + """Check for redundant cd ${S} statements""" + repoman_check_name = 'ebuild.minorsyn' + _src_phases = r'^\s*src_(prepare|configure|compile|install|test)\s*\(\)' + method_re = re.compile(_src_phases) + cds_re = re.compile(r'^\s*cd\s+("\$(\{S\}|S)"|\$(\{S\}|S))\s') + + def __init__(self): + self.check_next_line = False + + def check(self, num, line): + if self.check_next_line: + self.check_next_line = False + if self.cds_re.match(line): + return errors.REDUNDANT_CD_S_ERROR + elif self.method_re.match(line): + self.check_next_line = True + + +class EapiDefinition(LineCheck): + """ + Check that EAPI assignment conforms to PMS section 7.3.1 + (first non-comment, non-blank line). + """ + repoman_check_name = 'EAPI.definition' + ignore_comment = True + _eapi_re = portage._pms_eapi_re + + def new(self, pkg): + self._cached_eapi = pkg.eapi + self._parsed_eapi = None + self._eapi_line_num = None + + def check(self, num, line): + if self._eapi_line_num is None and line.strip(): + self._eapi_line_num = num + 1 + m = self._eapi_re.match(line) + if m is not None: + self._parsed_eapi = m.group(2) + + def end(self): + if self._parsed_eapi is None: + if self._cached_eapi != "0": + yield "valid EAPI assignment must occur on or before line: %s" % \ + self._eapi_line_num + elif self._parsed_eapi != self._cached_eapi: + yield ( + "bash returned EAPI '%s' which does not match " + "assignment on line: %s" % + (self._cached_eapi, self._eapi_line_num)) + + +class EbuildPatches(LineCheck): + """Ensure ebuilds use bash arrays for PATCHES to ensure white space safety""" + repoman_check_name = 'ebuild.patches' + re = re.compile(r'^\s*PATCHES=[^\(]') + error = errors.PATCHES_ERROR + + def check_eapi(self, eapi): + return eapi in ("0", "1", "2", "3", "4", "4-python", + "4-slot-abi", "5", "5-hdepend", "5-progress") + + +class EbuildQuotedA(LineCheck): + """Ensure ebuilds have no quoting around ${A}""" + + repoman_check_name = 'ebuild.minorsyn' + a_quoted = re.compile(r'.*\"\$(\{A\}|A)\"') + + def check(self, num, line): + match = self.a_quoted.match(line) + if match: + return "Quoted \"${A}\" on line: %d" + + +class NoOffsetWithHelpers(LineCheck): + """ Check that the image location, the alternate root offset, and the + offset prefix (D, ROOT, ED, EROOT and EPREFIX) are not used with + helpers """ + + repoman_check_name = 'variable.usedwithhelpers' + # Ignore matches in quoted strings like this: + # elog "installed into ${ROOT}usr/share/php5/apc/." + _install_funcs = ( + 'docinto|do(compress|dir|hard)' + '|exeinto|fowners|fperms|insinto|into') + _quoted_vars = 'D|ROOT|ED|EROOT|EPREFIX' + re = re.compile( + r'^[^#"\']*\b(%s)\s+"?\$\{?(%s)\b.*' % + (_install_funcs, _quoted_vars)) + error = errors.NO_OFFSET_WITH_HELPERS + + +class ImplicitRuntimeDeps(LineCheck): + """ + Detect the case where DEPEND is set and RDEPEND is unset in the ebuild, + since this triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4). + """ + + repoman_check_name = 'RDEPEND.implicit' + _assignment_re = re.compile(r'^\s*(R?DEPEND)\+?=') + + def new(self, pkg): + self._rdepend = False + self._depend = False + + def check_eapi(self, eapi): + # Beginning with EAPI 4, there is no + # implicit RDEPEND=$DEPEND assignment + # to be concerned with. + return eapi_has_implicit_rdepend(eapi) + + def check(self, num, line): + if not self._rdepend: + m = self._assignment_re.match(line) + if m is None: + pass + elif m.group(1) == "RDEPEND": + self._rdepend = True + elif m.group(1) == "DEPEND": + self._depend = True + + def end(self): + if self._depend and not self._rdepend: + yield 'RDEPEND is not explicitly assigned' + + +class InheritDeprecated(LineCheck): + """Check if ebuild directly or indirectly inherits a deprecated eclass.""" + + repoman_check_name = 'inherit.deprecated' + + # deprecated eclass : new eclass (False if no new eclass) + deprecated_eclasses = { + "base": False, + "bash-completion": "bash-completion-r1", + "boost-utils": False, + "distutils": "distutils-r1", + "games": False, + "gems": "ruby-fakegem", + "mono": "mono-env", + "python": "python-r1 / python-single-r1 / python-any-r1", + "ruby": "ruby-ng", + "x-modular": "xorg-2", + "gst-plugins-bad": "gstreamer", + "gst-plugins-base": "gstreamer", + "gst-plugins-good": "gstreamer", + "gst-plugins-ugly": "gstreamer", + "gst-plugins10": "gstreamer", + "clutter": "gnome2", + } + + _inherit_re = re.compile(r'^\s*inherit\s(.*)$') + + def new(self, pkg): + self._errors = [] + + def check(self, num, line): + direct_inherits = None + m = self._inherit_re.match(line) + if m is not None: + direct_inherits = m.group(1) + if direct_inherits: + direct_inherits = direct_inherits.split() + + if not direct_inherits: + return + + for eclass in direct_inherits: + replacement = self.deprecated_eclasses.get(eclass) + if replacement is None: + pass + elif replacement is False: + self._errors.append( + "please migrate from " + "'%s' (no replacement) on line: %d" % (eclass, num + 1)) + else: + self._errors.append( + "please migrate from " + "'%s' to '%s' on line: %d" % (eclass, replacement, num + 1)) + + def end(self): + for error in self._errors: + yield error + del self._errors + + + +class InheritEclass(LineCheck): + """ + Base class for checking for missing inherits, as well as excess inherits. + + Args: + eclass: Set to the name of your eclass. + funcs: A tuple of functions that this eclass provides. + comprehensive: Is the list of functions complete? + exempt_eclasses: If these eclasses are inherited, disable the missing + inherit check. + """ + + def __init__( + self, eclass, funcs=None, comprehensive=False, + exempt_eclasses=None, ignore_missing=False, **kwargs): + self._eclass = eclass + self._comprehensive = comprehensive + self._exempt_eclasses = exempt_eclasses + self._ignore_missing = ignore_missing + inherit_re = eclass + self._inherit_re = re.compile( + r'^(\s*|.*[|&]\s*)\binherit\s(.*\s)?%s(\s|$)' % inherit_re) + # Match when the function is preceded only by leading whitespace, a + # shell operator such as (, {, |, ||, or &&, or optional variable + # setting(s). This prevents false positives in things like elog + # messages, as reported in bug #413285. + self._func_re = re.compile( + r'(^|[|&{(])\s*(\w+=.*)?\b(' + '|'.join(funcs) + r')\b') + + def new(self, pkg): + self.repoman_check_name = 'inherit.missing' + # We can't use pkg.inherited because that tells us all the eclasses that + # have been inherited and not just the ones we inherit directly. + self._inherit = False + self._func_call = False + if self._exempt_eclasses is not None: + inherited = pkg.inherited + self._disabled = any(x in inherited for x in self._exempt_eclasses) + else: + self._disabled = False + self._eapi = pkg.eapi + + def check(self, num, line): + if not self._inherit: + self._inherit = self._inherit_re.match(line) + if not self._inherit: + if self._disabled or self._ignore_missing: + return + s = self._func_re.search(line) + if s is not None: + func_name = s.group(3) + eapi_func = _eclass_eapi_functions.get(func_name) + if eapi_func is None or not eapi_func(self._eapi): + self._func_call = True + return ( + '%s.eclass is not inherited, ' + 'but "%s" found at line: %s' % + (self._eclass, func_name, '%d')) + elif not self._func_call: + self._func_call = self._func_re.search(line) + + def end(self): + if not self._disabled and self._comprehensive and self._inherit \ + and not self._func_call: + self.repoman_check_name = 'inherit.unused' + yield 'no function called from %s.eclass; please drop' % self._eclass + +_usex_supported_eapis = ("0", "1", "2", "3", "4", "4-python", "4-slot-abi") +_in_iuse_supported_eapis = ("0", "1", "2", "3", "4", "4-python", "4-slot-abi", + "5", "5-hdepend", "5-progress") +_get_libdir_supported_eapis = _in_iuse_supported_eapis +_eclass_eapi_functions = { + "usex": lambda eapi: eapi not in _usex_supported_eapis, + "in_iuse": lambda eapi: eapi not in _in_iuse_supported_eapis, + "get_libdir": lambda eapi: eapi not in _get_libdir_supported_eapis, +} + +# eclasses that export ${ECLASS}_src_(compile|configure|install) +_eclass_export_functions = ( + 'ant-tasks', 'apache-2', 'apache-module', 'aspell-dict', + 'autotools-utils', 'base', 'bsdmk', 'cannadic', + 'clutter', 'cmake-utils', 'db', 'distutils', 'elisp', + 'embassy', 'emboss', 'emul-linux-x86', 'enlightenment', + 'font-ebdftopcf', 'font', 'fox', 'freebsd', 'freedict', + 'games', 'games-ggz', 'games-mods', 'gdesklets', + 'gems', 'gkrellm-plugin', 'gnatbuild', 'gnat', 'gnome2', + 'gnome-python-common', 'gnustep-base', 'go-mono', 'gpe', + 'gst-plugins-bad', 'gst-plugins-base', 'gst-plugins-good', + 'gst-plugins-ugly', 'gtk-sharp-module', 'haskell-cabal', + 'horde', 'java-ant-2', 'java-pkg-2', 'java-pkg-simple', + 'java-virtuals-2', 'kde4-base', 'kde4-meta', 'kernel-2', + 'latex-package', 'linux-mod', 'mozlinguas', 'myspell', + 'myspell-r2', 'mysql', 'mysql-v2', 'mythtv-plugins', + 'oasis', 'obs-service', 'office-ext', 'perl-app', + 'perl-module', 'php-ext-base-r1', 'php-ext-pecl-r2', + 'php-ext-source-r2', 'php-lib-r1', 'php-pear-lib-r1', + 'php-pear-r1', 'python-distutils-ng', 'python', + 'qt4-build', 'qt4-r2', 'rox-0install', 'rox', 'ruby', + 'ruby-ng', 'scsh', 'selinux-policy-2', 'sgml-catalog', + 'stardict', 'sword-module', 'tetex-3', 'tetex', + 'texlive-module', 'toolchain-binutils', 'toolchain', + 'twisted', 'vdr-plugin-2', 'vdr-plugin', 'vim', + 'vim-plugin', 'vim-spell', 'virtuoso', 'vmware', + 'vmware-mod', 'waf-utils', 'webapp', 'xemacs-elisp', + 'xemacs-packages', 'xfconf', 'x-modular', 'xorg-2', + 'zproduct' +) + +_eclass_info = { + 'autotools': { + 'funcs': ( + 'eaclocal', 'eautoconf', 'eautoheader', + 'eautomake', 'eautoreconf', '_elibtoolize', + 'eautopoint' + ), + 'comprehensive': True, + + # Exempt eclasses: + # git - An EGIT_BOOTSTRAP variable may be used to call one of + # the autotools functions. + # subversion - An ESVN_BOOTSTRAP variable may be used to call one of + # the autotools functions. + 'exempt_eclasses': ('git', 'git-2', 'subversion', 'autotools-utils') + }, + + 'eutils': { + 'funcs': ( + 'estack_push', 'estack_pop', 'eshopts_push', 'eshopts_pop', + 'eumask_push', 'eumask_pop', 'epatch', 'epatch_user', + 'emktemp', 'edos2unix', 'in_iuse', 'use_if_iuse', 'usex' + ), + 'comprehensive': False, + + # These are "eclasses are the whole ebuild" type thing. + 'exempt_eclasses': _eclass_export_functions, + }, + + 'flag-o-matic': { + 'funcs': ( + 'filter-(ld)?flags', 'strip-flags', 'strip-unsupported-flags', + 'append-((ld|c(pp|xx)?))?flags', 'append-libs', + ), + 'comprehensive': False + }, + + 'libtool': { + 'funcs': ( + 'elibtoolize', + ), + 'comprehensive': True, + 'exempt_eclasses': ('autotools',) + }, + + 'multilib': { + 'funcs': ( + 'get_libdir', + ), + + # These are "eclasses are the whole ebuild" type thing. + 'exempt_eclasses': _eclass_export_functions + ( + 'autotools', 'libtool', 'multilib-minimal'), + + 'comprehensive': False + }, + + 'multiprocessing': { + 'funcs': ( + 'makeopts_jobs', + ), + 'comprehensive': False + }, + + 'prefix': { + 'funcs': ( + 'eprefixify', + ), + 'comprehensive': True + }, + + 'toolchain-funcs': { + 'funcs': ( + 'gen_usr_ldscript', + ), + 'comprehensive': False + }, + + 'user': { + 'funcs': ( + 'enewuser', 'enewgroup', + 'egetent', 'egethome', 'egetshell', 'esethome' + ), + 'comprehensive': True + } +} + + +class EMakeParallelDisabled(PhaseCheck): + """Check for emake -j1 calls which disable parallelization.""" + repoman_check_name = 'upstream.workaround' + re = re.compile(r'^\s*emake\s+.*-j\s*1\b') + error = errors.EMAKE_PARALLEL_DISABLED + + def phase_check(self, num, line): + if self.in_phase == 'src_compile' or self.in_phase == 'src_install': + if self.re.match(line): + return self.error + + +class EMakeParallelDisabledViaMAKEOPTS(LineCheck): + """Check for MAKEOPTS=-j1 that disables parallelization.""" + repoman_check_name = 'upstream.workaround' + re = re.compile(r'^\s*MAKEOPTS=(\'|")?.*-j\s*1\b') + error = errors.EMAKE_PARALLEL_DISABLED_VIA_MAKEOPTS + + +class NoAsNeeded(LineCheck): + """Check for calls to the no-as-needed function.""" + repoman_check_name = 'upstream.workaround' + re = re.compile(r'.*\$\(no-as-needed\)') + error = errors.NO_AS_NEEDED + + +class PreserveOldLib(LineCheck): + """Check for calls to the deprecated preserve_old_lib function.""" + repoman_check_name = 'ebuild.minorsyn' + re = re.compile(r'.*preserve_old_lib') + error = errors.PRESERVE_OLD_LIB + + +class SandboxAddpredict(LineCheck): + """Check for calls to the addpredict function.""" + repoman_check_name = 'upstream.workaround' + re = re.compile(r'(^|\s)addpredict\b') + error = errors.SANDBOX_ADDPREDICT + + +class DeprecatedBindnowFlags(LineCheck): + """Check for calls to the deprecated bindnow-flags function.""" + repoman_check_name = 'ebuild.minorsyn' + re = re.compile(r'.*\$\(bindnow-flags\)') + error = errors.DEPRECATED_BINDNOW_FLAGS + + +class WantAutoDefaultValue(LineCheck): + """Check setting WANT_AUTO* to latest (default value).""" + repoman_check_name = 'ebuild.minorsyn' + _re = re.compile(r'^WANT_AUTO(CONF|MAKE)=(\'|")?latest') + + def check(self, num, line): + m = self._re.match(line) + if m is not None: + return 'WANT_AUTO' + m.group(1) + \ + ' redundantly set to default value "latest" on line: %d' + + +class SrcCompileEconf(PhaseCheck): + repoman_check_name = 'ebuild.minorsyn' + configure_re = re.compile(r'\s(econf|./configure)') + + def check_eapi(self, eapi): + return eapi_has_src_prepare_and_src_configure(eapi) + + def phase_check(self, num, line): + if self.in_phase == 'src_compile': + m = self.configure_re.match(line) + if m is not None: + return ("'%s'" % m.group(1)) + \ + " call should be moved to src_configure from line: %d" + + +class SrcUnpackPatches(PhaseCheck): + repoman_check_name = 'ebuild.minorsyn' + src_prepare_tools_re = re.compile(r'\s(e?patch|sed)\s') + + def check_eapi(self, eapi): + return eapi_has_src_prepare_and_src_configure(eapi) + + def phase_check(self, num, line): + if self.in_phase == 'src_unpack': + m = self.src_prepare_tools_re.search(line) + if m is not None: + return ("'%s'" % m.group(1)) + \ + " call should be moved to src_prepare from line: %d" + + +class BuiltWithUse(LineCheck): + repoman_check_name = 'ebuild.minorsyn' + re = re.compile(r'(^|.*\b)built_with_use\b') + error = errors.BUILT_WITH_USE + + +class DeprecatedUseq(LineCheck): + """Checks for use of the deprecated useq function""" + repoman_check_name = 'ebuild.minorsyn' + re = re.compile(r'(^|.*\b)useq\b') + error = errors.USEQ_ERROR + + +class DeprecatedHasq(LineCheck): + """Checks for use of the deprecated hasq function""" + repoman_check_name = 'ebuild.minorsyn' + re = re.compile(r'(^|.*\b)hasq\b') + error = errors.HASQ_ERROR + + +# EAPI <2 checks +class UndefinedSrcPrepareSrcConfigurePhases(LineCheck): + repoman_check_name = 'EAPI.incompatible' + src_configprepare_re = re.compile(r'\s*(src_configure|src_prepare)\s*\(\)') + + def check_eapi(self, eapi): + return not eapi_has_src_prepare_and_src_configure(eapi) + + def check(self, num, line): + m = self.src_configprepare_re.match(line) + if m is not None: + return ("'%s'" % m.group(1)) + \ + " phase is not defined in EAPI < 2 on line: %d" + + +# EAPI-3 checks +class Eapi3DeprecatedFuncs(LineCheck): + repoman_check_name = 'EAPI.deprecated' + deprecated_commands_re = re.compile(r'^\s*(check_license)\b') + + def check_eapi(self, eapi): + return eapi not in ('0', '1', '2') + + def check(self, num, line): + m = self.deprecated_commands_re.match(line) + if m is not None: + return ("'%s'" % m.group(1)) + \ + " has been deprecated in EAPI=3 on line: %d" + + +# EAPI <4 checks +class UndefinedPkgPretendPhase(LineCheck): + repoman_check_name = 'EAPI.incompatible' + pkg_pretend_re = re.compile(r'\s*(pkg_pretend)\s*\(\)') + + def check_eapi(self, eapi): + return not eapi_has_pkg_pretend(eapi) + + def check(self, num, line): + m = self.pkg_pretend_re.match(line) + if m is not None: + return ("'%s'" % m.group(1)) + \ + " phase is not defined in EAPI < 4 on line: %d" + + +# EAPI-4 checks +class Eapi4IncompatibleFuncs(LineCheck): + repoman_check_name = 'EAPI.incompatible' + banned_commands_re = re.compile(r'^\s*(dosed|dohard)') + + def check_eapi(self, eapi): + return not eapi_has_dosed_dohard(eapi) + + def check(self, num, line): + m = self.banned_commands_re.match(line) + if m is not None: + return ("'%s'" % m.group(1)) + \ + " has been banned in EAPI=4 on line: %d" + + +class Eapi4GoneVars(LineCheck): + repoman_check_name = 'EAPI.incompatible' + undefined_vars_re = re.compile( + r'.*\$(\{(AA|KV|EMERGE_FROM)\}|(AA|KV|EMERGE_FROM))') + + def check_eapi(self, eapi): + # AA, KV, and EMERGE_FROM should not be referenced in EAPI 4 or later. + return not eapi_exports_AA(eapi) + + def check(self, num, line): + m = self.undefined_vars_re.match(line) + if m is not None: + return ("variable '$%s'" % m.group(1)) + \ + " is gone in EAPI=4 on line: %d" + + +class PortageInternal(LineCheck): + repoman_check_name = 'portage.internal' + ignore_comment = True + # Match when the command is preceded only by leading whitespace or a shell + # operator such as (, {, |, ||, or &&. This prevents false positives in + # things like elog messages, as reported in bug #413285. + + internal_portage_func_or_var = ( + 'ecompress|ecompressdir|env-update|prepall|prepalldocs|preplib') + re = re.compile( + r'^(\s*|.*[|&{(]+\s*)\b(%s)\b' % internal_portage_func_or_var) + + def check(self, num, line): + """Run the check on line and return error if there is one""" + m = self.re.match(line) + if m is not None: + return ("'%s'" % m.group(2)) + " called on line: %d" + + +class PortageInternalVariableAssignment(LineCheck): + repoman_check_name = 'portage.internal' + internal_assignment = re.compile( + r'\s*(export\s+)?(EXTRA_ECONF|EXTRA_EMAKE)\+?=') + + def check(self, num, line): + match = self.internal_assignment.match(line) + e = None + if match is not None: + e = 'Assignment to variable %s' % match.group(2) + e += ' on line: %d' + return e + +_base_check_classes = (InheritEclass, LineCheck, PhaseCheck) +_constant_checks = None + + +def checks_init(experimental_inherit=False): + + global _constant_checks, _eclass_info + + if not experimental_inherit: + # Emulate the old eprefixify.defined and inherit.autotools checks. + _eclass_info = { + 'autotools': { + 'funcs': ( + 'eaclocal', 'eautoconf', 'eautoheader', + 'eautomake', 'eautoreconf', '_elibtoolize', + 'eautopoint' + ), + 'comprehensive': True, + 'ignore_missing': True, + 'exempt_eclasses': ('git', 'git-2', 'subversion', 'autotools-utils') + }, + + 'prefix': { + 'funcs': ( + 'eprefixify', + ), + 'comprehensive': False + } + } + + _constant_checks = tuple( + chain(( + v() for k, v in globals().items() + if ( + isinstance(v, type) + and issubclass(v, LineCheck) + and v not in _base_check_classes)), ( + InheritEclass(k, **portage._native_kwargs(kwargs)) + for k, kwargs in _eclass_info.items()))) + + +_here_doc_re = re.compile(r'.*<<[-]?(\w+)\s*(>\s*\S+\s*)?$') +_ignore_comment_re = re.compile(r'^\s*#') + + +def run_checks(contents, pkg): + unicode_escape_codec = codecs.lookup('unicode_escape') + unicode_escape = lambda x: unicode_escape_codec.decode(x)[0] + if _constant_checks is None: + checks_init() + checks = _constant_checks + here_doc_delim = None + multiline = None + + for lc in checks: + lc.new(pkg) + + multinum = 0 + for num, line in enumerate(contents): + + # Check if we're inside a here-document. + if here_doc_delim is not None: + if here_doc_delim.match(line): + here_doc_delim = None + if here_doc_delim is None: + here_doc = _here_doc_re.match(line) + if here_doc is not None: + here_doc_delim = re.compile(r'^\s*%s$' % here_doc.group(1)) + if here_doc_delim is not None: + continue + + # Unroll multiline escaped strings so that we can check things: + # inherit foo bar \ + # moo \ + # cow + # This will merge these lines like so: + # inherit foo bar moo cow + try: + # A normal line will end in the two bytes: <\> <\n>. So decoding + # that will result in python thinking the <\n> is being escaped + # and eat the single <\> which makes it hard for us to detect. + # Instead, strip the newline (which we know all lines have), and + # append a <0>. Then when python escapes it, if the line ended + # in a <\>, we'll end up with a <\0> marker to key off of. This + # shouldn't be a problem with any valid ebuild ... + line_escaped = unicode_escape(line.rstrip('\n') + '0') + except SystemExit: + raise + except: + # Who knows what kind of crazy crap an ebuild will have + # in it -- don't allow it to kill us. + line_escaped = line + if multiline: + # Chop off the \ and \n bytes from the previous line. + multiline = multiline[:-2] + line + if not line_escaped.endswith('\0'): + line = multiline + num = multinum + multiline = None + else: + continue + else: + if line_escaped.endswith('\0'): + multinum = num + multiline = line + continue + + if not line.endswith("#nowarn\n"): + # Finally we have a full line to parse. + is_comment = _ignore_comment_re.match(line) is not None + for lc in checks: + if is_comment and lc.ignore_comment: + continue + if lc.check_eapi(pkg.eapi): + ignore = lc.ignore_line + if not ignore or not ignore.match(line): + e = lc.check(num, line) + if e: + yield lc.repoman_check_name, e % (num + 1) + + for lc in checks: + i = lc.end() + if i is not None: + for e in i: + yield lc.repoman_check_name, e diff --git a/repoman/pym/repoman/modules/scan/ebuild/ebuild.py b/repoman/pym/repoman/modules/scan/ebuild/ebuild.py new file mode 100644 index 000000000..28cb8b407 --- /dev/null +++ b/repoman/pym/repoman/modules/scan/ebuild/ebuild.py @@ -0,0 +1,238 @@ +# -*- coding:utf-8 -*- + +from __future__ import print_function, unicode_literals + +import re +import stat + +from _emerge.Package import Package +from _emerge.RootConfig import RootConfig + +from repoman.modules.scan.scanbase import ScanBase +from repoman.qa_data import no_exec, allvars +# import our initialized portage instance +from repoman._portage import portage +from portage import os +from portage.const import LIVE_ECLASSES +from portage.exception import InvalidPackageName + +pv_toolong_re = re.compile(r'[0-9]{19,}') + + +class Ebuild(ScanBase): + '''Class to run primary checks on ebuilds''' + + def __init__(self, **kwargs): + '''Class init + + @param qatracker: QATracker instance + @param portdb: portdb instance + @param repo_settings: repository settings instance + @param vcs_settings: VCSSettings instance + @param checks: checks dictionary + ''' + super(Ebuild, self).__init__(**kwargs) + self.qatracker = kwargs.get('qatracker') + self.portdb = kwargs.get('portdb') + self.repo_settings = kwargs.get('repo_settings') + self.vcs_settings = kwargs.get('vcs_settings') + self.checks = kwargs.get('checks') + self.root_config = RootConfig(self.repo_settings.repoman_settings, + self.repo_settings.trees[self.repo_settings.root], None) + self.changed = None + self.xpkg = None + self.y_ebuild = None + self.pkg = None + self.metadata = None + self.eapi = None + self.inherited = None + self.live_ebuild = None + self.keywords = None + self.pkgs = {} + + def _set_paths(self, **kwargs): + repolevel = kwargs.get('repolevel') + self.relative_path = os.path.join(self.xpkg, self.y_ebuild + ".ebuild") + self.full_path = os.path.join(self.repo_settings.repodir, self.relative_path) + self.ebuild_path = self.y_ebuild + ".ebuild" + if repolevel < 3: + self.ebuild_path = os.path.join(kwargs.get('pkgdir'), self.ebuild_path) + if repolevel < 2: + self.ebuild_path = os.path.join(kwargs.get('catdir'), self.ebuild_path) + self.ebuild_path = os.path.join(".", self.ebuild_path) + + @property + def untracked(self): + '''Determines and returns if the ebuild is not tracked by the vcs''' + do_check = self.vcs_settings.vcs in ("cvs", "svn", "bzr") + really_notadded = (self.checks['ebuild_notadded'] and + self.y_ebuild not in self.vcs_settings.eadded) + if do_check and really_notadded: + # ebuild not added to vcs + return True + return False + + def check(self, **kwargs): + '''Perform a changelog and untracked checks on the ebuild + + @param xpkg: Package in which we check (object). + @param y_ebuild: Ebuild which we check (string). + @param changed: dictionary instance + @param repolevel: The depth within the repository + @param catdir: The category directiory + @param pkgdir: the package directory + @returns: dictionary, including {ebuild object} + ''' + self.xpkg = kwargs.get('xpkg') + self.y_ebuild = kwargs.get('y_ebuild') + self.changed = kwargs.get('changed') + changelog_modified = kwargs.get('changelog_modified') + self._set_paths(**kwargs) + + if self.checks['changelog'] and not changelog_modified \ + and self.ebuild_path in self.changed.new_ebuilds: + self.qatracker.add_error('changelog.ebuildadded', self.relative_path) + + if self.untracked: + # ebuild not added to vcs + self.qatracker.add_error( + "ebuild.notadded", self.xpkg + "/" + self.y_ebuild + ".ebuild") + # update the dynamic data + dyn_ebuild = kwargs.get('ebuild') + dyn_ebuild.set(self) + return False + + def set_pkg_data(self, **kwargs): + '''Sets some classwide data needed for some of the checks + + @returns: dictionary + ''' + self.pkg = self.pkgs[self.y_ebuild] + self.metadata = self.pkg._metadata + self.eapi = self.metadata["EAPI"] + self.inherited = self.pkg.inherited + self.live_ebuild = LIVE_ECLASSES.intersection(self.inherited) + self.keywords = self.metadata["KEYWORDS"].split() + self.archs = set(kw.lstrip("~") for kw in self.keywords if not kw.startswith("-")) + return False + + def bad_split_check(self, **kwargs): + '''Checks for bad category/package splits. + + @param pkgdir: string: path + @returns: dictionary + ''' + pkgdir = kwargs.get('pkgdir') + myesplit = portage.pkgsplit(self.y_ebuild) + is_bad_split = myesplit is None or myesplit[0] != self.xpkg.split("/")[-1] + if is_bad_split: + is_pv_toolong = pv_toolong_re.search(myesplit[1]) + is_pv_toolong2 = pv_toolong_re.search(myesplit[2]) + if is_pv_toolong or is_pv_toolong2: + self.qatracker.add_error( + "ebuild.invalidname", self.xpkg + "/" + self.y_ebuild + ".ebuild") + return True + elif myesplit[0] != pkgdir: + print(pkgdir, myesplit[0]) + self.qatracker.add_error( + "ebuild.namenomatch", self.xpkg + "/" + self.y_ebuild + ".ebuild") + return True + return False + + def pkg_invalid(self, **kwargs): + '''Sets some pkg info and checks for invalid packages + + @param validity_future: Future instance + @returns: dictionary, including {pkg object} + ''' + fuse = kwargs.get('validity_future') + dyn_pkg = kwargs.get('pkg') + if self.pkg.invalid: + for k, msgs in self.pkg.invalid.items(): + for msg in msgs: + self.qatracker.add_error(k, "%s: %s" % (self.relative_path, msg)) + # update the dynamic data + fuse.set(False, ignore_InvalidState=True) + dyn_pkg.set(self.pkg) + return True + # update the dynamic data + dyn_pkg.set(self.pkg) + return False + + def check_isebuild(self, **kwargs): + '''Test the file for qualifications that is is an ebuild + + @param checkdirlist: list of files in the current package directory + @param checkdir: current package directory path + @param xpkg: current package directory being checked + @param validity_future: Future instance + @returns: dictionary, including {pkgs, can_force} + ''' + checkdirlist = kwargs.get('checkdirlist').get() + checkdir = kwargs.get('checkdir') + xpkg = kwargs.get('xpkg') + fuse = kwargs.get('validity_future') + can_force = kwargs.get('can_force') + self.continue_ = False + ebuildlist = [] + pkgs = {} + for y in checkdirlist: + file_is_ebuild = y.endswith(".ebuild") + file_should_be_non_executable = y in no_exec or file_is_ebuild + + if file_should_be_non_executable: + file_is_executable = stat.S_IMODE( + os.stat(os.path.join(checkdir, y)).st_mode) & 0o111 + + if file_is_executable: + self.qatracker.add_error("file.executable", os.path.join(checkdir, y)) + if file_is_ebuild: + pf = y[:-7] + ebuildlist.append(pf) + catdir = xpkg.split("/")[0] + cpv = "%s/%s" % (catdir, pf) + try: + myaux = dict(zip(allvars, self.portdb.aux_get(cpv, allvars))) + except KeyError: + fuse.set(False, ignore_InvalidState=True) + self.qatracker.add_error("ebuild.syntax", os.path.join(xpkg, y)) + continue + except IOError: + fuse.set(False, ignore_InvalidState=True) + self.qatracker.add_error("ebuild.output", os.path.join(xpkg, y)) + continue + except InvalidPackageName: + fuse.set(False, ignore_InvalidState=True) + self.qatracker.add_error("ebuild.invalidname", os.path.join(xpkg, y)) + continue + if not portage.eapi_is_supported(myaux["EAPI"]): + fuse.set(False, ignore_InvalidState=True) + self.qatracker.add_error("EAPI.unsupported", os.path.join(xpkg, y)) + continue + pkgs[pf] = Package( + cpv=cpv, metadata=myaux, root_config=self.root_config, + type_name="ebuild") + + if len(pkgs) != len(ebuildlist): + # If we can't access all the metadata then it's totally unsafe to + # commit since there's no way to generate a correct Manifest. + # Do not try to do any more QA checks on this package since missing + # metadata leads to false positives for several checks, and false + # positives confuse users. + self.continue_ = True + can_force.set(False, ignore_InvalidState=True) + self.pkgs = pkgs + # set our updated data + dyn_pkgs = kwargs.get('pkgs') + dyn_pkgs.set(pkgs) + return self.continue_ + + @property + def runInPkgs(self): + '''Package level scans''' + return (True, [self.check_isebuild]) + + @property + def runInEbuilds(self): + '''Ebuild level scans''' + return (True, [self.check, self.set_pkg_data, self.bad_split_check, self.pkg_invalid]) diff --git a/repoman/pym/repoman/modules/scan/ebuild/errors.py b/repoman/pym/repoman/modules/scan/ebuild/errors.py new file mode 100644 index 000000000..3090de0d1 --- /dev/null +++ b/repoman/pym/repoman/modules/scan/ebuild/errors.py @@ -0,0 +1,49 @@ +# -*- coding:utf-8 -*- +# repoman: Error Messages +# Copyright 2007-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import unicode_literals + +COPYRIGHT_ERROR = ( + 'Invalid Gentoo Copyright on line: %d') +LICENSE_ERROR = ( + 'Invalid Gentoo/GPL License on line: %d') +ID_HEADER_ERROR = ( + 'Malformed Id header on line: %d') +LEADING_SPACES_ERROR = ( + 'Ebuild contains leading spaces on line: %d') +TRAILING_WHITESPACE_ERROR = ( + 'Trailing whitespace error on line: %d') +READONLY_ASSIGNMENT_ERROR = ( + 'Ebuild contains assignment to read-only variable on line: %d') +MISSING_QUOTES_ERROR = ( + 'Unquoted Variable on line: %d') +NESTED_DIE_ERROR = ( + 'Ebuild calls die in a subshell on line: %d') +PATCHES_ERROR = ( + 'PATCHES is not a bash array on line: %d') +REDUNDANT_CD_S_ERROR = ( + 'Ebuild has redundant cd ${S} statement on line: %d') +EMAKE_PARALLEL_DISABLED = ( + 'Upstream parallel compilation bug (ebuild calls emake -j1 on line: %d)') +EMAKE_PARALLEL_DISABLED_VIA_MAKEOPTS = ( + 'Upstream parallel compilation bug (MAKEOPTS=-j1 on line: %d)') +DEPRECATED_BINDNOW_FLAGS = ( + 'Deprecated bindnow-flags call on line: %d') +EAPI_DEFINED_AFTER_INHERIT = ( + 'EAPI defined after inherit on line: %d') +NO_AS_NEEDED = ( + 'Upstream asneeded linking bug (no-as-needed on line: %d)') +PRESERVE_OLD_LIB = ( + 'Ebuild calls deprecated preserve_old_lib on line: %d') +BUILT_WITH_USE = ( + 'built_with_use on line: %d') +NO_OFFSET_WITH_HELPERS = ( + "Helper function is used with D, ROOT, ED, EROOT or EPREFIX on line :%d") +SANDBOX_ADDPREDICT = ( + 'Ebuild calls addpredict on line: %d') +USEQ_ERROR = ( + 'Ebuild calls deprecated useq function on line: %d') +HASQ_ERROR = ( + 'Ebuild calls deprecated hasq function on line: %d') diff --git a/repoman/pym/repoman/modules/scan/ebuild/multicheck.py b/repoman/pym/repoman/modules/scan/ebuild/multicheck.py new file mode 100644 index 000000000..9e36e2a68 --- /dev/null +++ b/repoman/pym/repoman/modules/scan/ebuild/multicheck.py @@ -0,0 +1,56 @@ + +'''multicheck.py +Perform multiple different checks on an ebuild +''' + +import io + +from portage import _encodings, _unicode_encode + +from repoman.modules.scan.scanbase import ScanBase +from .checks import run_checks, checks_init + + +class MultiCheck(ScanBase): + '''Class to run multiple different checks on an ebuild''' + + def __init__(self, **kwargs): + '''Class init + + @param qatracker: QATracker instance + @param options: the run time cli options + ''' + self.qatracker = kwargs.get('qatracker') + self.options = kwargs.get('options') + checks_init(self.options.experimental_inherit == 'y') + + def check(self, **kwargs): + '''Check the ebuild for utf-8 encoding + + @param pkg: Package in which we check (object). + @param ebuild: Ebuild which we check (object). + @returns: dictionary + ''' + ebuild = kwargs.get('ebuild').get() + pkg = kwargs.get('pkg').get() + try: + # All ebuilds should have utf_8 encoding. + f = io.open( + _unicode_encode(ebuild.full_path, encoding=_encodings['fs'], + errors='strict'), + mode='r', encoding=_encodings['repo.content']) + try: + for check_name, e in run_checks(f, pkg): + self.qatracker.add_error( + check_name, ebuild.relative_path + ': %s' % e) + finally: + f.close() + except UnicodeDecodeError: + # A file.UTF8 failure will have already been recorded. + pass + return False + + @property + def runInEbuilds(self): + '''Ebuild level scans''' + return (True, [self.check]) diff --git a/repoman/pym/repoman/modules/scan/eclasses/__init__.py b/repoman/pym/repoman/modules/scan/eclasses/__init__.py new file mode 100644 index 000000000..78d46e4b4 --- /dev/null +++ b/repoman/pym/repoman/modules/scan/eclasses/__init__.py @@ -0,0 +1,47 @@ +# Copyright 2015-2016 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +doc = """Eclasses plug-in module for repoman. +Performs an live and ruby eclass checks on ebuilds.""" +__doc__ = doc[:] + + +module_spec = { + 'name': 'eclasses', + 'description': doc, + 'provides':{ + 'live-module': { + 'name': "live", + 'sourcefile': "live", + 'class': "LiveEclassChecks", + 'description': doc, + 'functions': ['check'], + 'func_kwargs': { + }, + 'mod_kwargs': ['qatracker', 'repo_metadata', 'repo_settings', + ], + 'func_kwargs': { + 'ebuild': (None, None), + 'pkg': (None, None), + 'xpkg': (None, None), + 'y_ebuild': (None, None), + }, + }, + 'ruby-module': { + 'name': "ruby", + 'sourcefile': "ruby", + 'class': "RubyEclassChecks", + 'description': doc, + 'functions': ['check'], + 'func_kwargs': { + }, + 'mod_kwargs': ['qatracker' + ], + 'func_kwargs': { + 'ebuild': (None, None), + 'pkg': (None, None), + }, + }, + } +} + diff --git a/repoman/pym/repoman/modules/scan/eclasses/live.py b/repoman/pym/repoman/modules/scan/eclasses/live.py new file mode 100644 index 000000000..dca10b583 --- /dev/null +++ b/repoman/pym/repoman/modules/scan/eclasses/live.py @@ -0,0 +1,76 @@ + +'''live.py +Performs Live eclass checks +''' + +from repoman._portage import portage +from repoman.modules.scan.scanbase import ScanBase + + +class LiveEclassChecks(ScanBase): + '''Performs checks for the usage of Live eclasses in ebuilds''' + + def __init__(self, **kwargs): + ''' + @param qatracker: QATracker instance + ''' + self.qatracker = kwargs.get('qatracker') + self.pmaskdict = kwargs.get('repo_metadata')['pmaskdict'] + self.repo_settings = kwargs.get('repo_settings') + + def check(self, **kwargs): + '''Ebuilds that inherit a "Live" eclass (darcs, subversion, git, cvs, + etc..) should not be allowed to be marked stable + + @param pkg: Package in which we check (object). + @param xpkg: Package in which we check (string). + @param ebuild: Ebuild which we check (object). + @param y_ebuild: Ebuild which we check (string). + @returns: boolean + ''' + pkg = kwargs.get("pkg").result() + package = kwargs.get('xpkg') + ebuild = kwargs.get('ebuild').get() + y_ebuild = kwargs.get('y_ebuild') + + if ebuild.live_ebuild and self.repo_settings.repo_config.name == "gentoo": + return self.check_live(pkg, package, ebuild, y_ebuild) + return False + + def check_live(self, pkg, package, ebuild, y_ebuild): + '''Perform the live vcs check + + @param pkg: Package in which we check (object). + @param xpkg: Package in which we check (string). + @param ebuild: Ebuild which we check (object). + @param y_ebuild: Ebuild which we check (string). + @returns: boolean + ''' + keywords = ebuild.keywords + is_stable = lambda kw: not kw.startswith("~") and not kw.startswith("-") + bad_stable_keywords = list(filter(is_stable, keywords)) + + if bad_stable_keywords: + self.qatracker.add_error( + "LIVEVCS.stable", "%s/%s.ebuild with stable keywords: %s" % ( + package, y_ebuild, bad_stable_keywords)) + + good_keywords_exist = len(bad_stable_keywords) < len(keywords) + if good_keywords_exist and not self._has_global_mask(pkg, self.pmaskdict): + self.qatracker.add_error("LIVEVCS.unmasked", ebuild.relative_path) + return False + + @staticmethod + def _has_global_mask(pkg, global_pmaskdict): + mask_atoms = global_pmaskdict.get(pkg.cp) + if mask_atoms: + pkg_list = [pkg] + for x in mask_atoms: + if portage.dep.match_from_list(x, pkg_list): + return x + return None + + @property + def runInEbuilds(self): + '''Ebuild level scans''' + return (True, [self.check]) diff --git a/repoman/pym/repoman/modules/scan/eclasses/ruby.py b/repoman/pym/repoman/modules/scan/eclasses/ruby.py new file mode 100644 index 000000000..b3501805e --- /dev/null +++ b/repoman/pym/repoman/modules/scan/eclasses/ruby.py @@ -0,0 +1,48 @@ + +'''ruby.py +Performs Ruby eclass checks +''' + +from repoman.qa_data import ruby_deprecated +from repoman.modules.scan.scanbase import ScanBase + + +class RubyEclassChecks(ScanBase): + '''Performs checks for the usage of Ruby eclasses in ebuilds''' + + def __init__(self, **kwargs): + ''' + @param qatracker: QATracker instance + ''' + super(RubyEclassChecks, self).__init__(**kwargs) + self.qatracker = kwargs.get('qatracker') + self.old_ruby_eclasses = ["ruby-ng", "ruby-fakegem", "ruby"] + + def check(self, **kwargs): + '''Check ebuilds that inherit the ruby eclasses + + @param pkg: Package in which we check (object). + @param ebuild: Ebuild which we check (object). + @returns: dictionary + ''' + pkg = kwargs.get('pkg').get() + ebuild = kwargs.get('ebuild').get() + is_inherited = lambda eclass: eclass in pkg.inherited + is_old_ruby_eclass_inherited = filter( + is_inherited, self.old_ruby_eclasses) + + if is_old_ruby_eclass_inherited: + ruby_intersection = pkg.iuse.all.intersection(ruby_deprecated) + + if ruby_intersection: + for myruby in ruby_intersection: + self.qatracker.add_error( + "IUSE.rubydeprecated", + (ebuild.relative_path + ": Deprecated ruby target: %s") + % myruby) + return False + + @property + def runInEbuilds(self): + '''Ebuild level scans''' + return (True, [self.check]) diff --git a/repoman/pym/repoman/modules/scan/fetch/__init__.py b/repoman/pym/repoman/modules/scan/fetch/__init__.py new file mode 100644 index 000000000..3c8e6002c --- /dev/null +++ b/repoman/pym/repoman/modules/scan/fetch/__init__.py @@ -0,0 +1,33 @@ +# Copyright 2015-2016 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +doc = """fetches plug-in module for repoman. +Performs fetch related checks on ebuilds.""" +__doc__ = doc[:] + + +module_spec = { + 'name': 'fetches', + 'description': doc, + 'provides':{ + 'fetches-module': { + 'name': "fetches", + 'sourcefile': "fetches", + 'class': "FetchChecks", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + 'mod_kwargs': ['portdb', 'qatracker', 'repo_settings', 'vcs_settings', + ], + 'func_kwargs': { + 'changed': (None, None), + 'checkdir': (None, None), + 'checkdir_relative': (None, None), + 'ebuild': (None, None), + 'xpkg': (None, None), + }, + }, + } +} + diff --git a/repoman/pym/repoman/modules/scan/fetch/fetches.py b/repoman/pym/repoman/modules/scan/fetch/fetches.py new file mode 100644 index 000000000..555f34f14 --- /dev/null +++ b/repoman/pym/repoman/modules/scan/fetch/fetches.py @@ -0,0 +1,190 @@ +# -*- coding:utf-8 -*- + +'''fetches.py +Performs the src_uri fetchlist and files checks +''' + +from stat import S_ISDIR + +# import our initialized portage instance +from repoman._portage import portage +from repoman.modules.vcs.vcs import vcs_new_changed +from repoman.modules.scan.scanbase import ScanBase + +from portage import os + + +class FetchChecks(ScanBase): + '''Performs checks on the files needed for the ebuild''' + + def __init__(self, **kwargs): + ''' + @param portdb: portdb instance + @param qatracker: QATracker instance + @param repo_settings: repository settings instance + @param vcs_settings: VCSSettings instance + ''' + super(FetchChecks, self).__init__(**kwargs) + self.portdb = kwargs.get('portdb') + self.qatracker = kwargs.get('qatracker') + self.repo_settings = kwargs.get('repo_settings') + self.repoman_settings = self.repo_settings.repoman_settings + self.vcs_settings = kwargs.get('vcs_settings') + self._src_uri_error = False + + # TODO: Build a regex instead here, for the SRC_URI.mirror check. + self.thirdpartymirrors = {} + profile_thirdpartymirrors = self.repo_settings.repoman_settings.thirdpartymirrors().items() + for mirror_alias, mirrors in profile_thirdpartymirrors: + for mirror in mirrors: + if not mirror.endswith("/"): + mirror += "/" + self.thirdpartymirrors[mirror] = mirror_alias + + def check(self, **kwargs): + '''Checks the ebuild sources and files for errors + + @param xpkg: the pacakge being checked + @param checkdir: string, directory path + @param checkdir_relative: repolevel determined path + @returns: boolean + ''' + xpkg = kwargs.get('xpkg') + checkdir = kwargs.get('checkdir') + checkdir_relative = kwargs.get('checkdir_relative') + changed = kwargs.get('changed').changed + new = kwargs.get('changed').new + _digests = self.digests(checkdir) + fetchlist_dict = portage.FetchlistDict( + checkdir, self.repoman_settings, self.portdb) + myfiles_all = [] + self._src_uri_error = False + for mykey in fetchlist_dict: + try: + myfiles_all.extend(fetchlist_dict[mykey]) + except portage.exception.InvalidDependString as e: + self._src_uri_error = True + try: + self.portdb.aux_get(mykey, ["SRC_URI"]) + except KeyError: + # This will be reported as an "ebuild.syntax" error. + pass + else: + self.qatracker.add_error( + "SRC_URI.syntax", "%s.ebuild SRC_URI: %s" % (mykey, e)) + del fetchlist_dict + if not self._src_uri_error: + # This test can produce false positives if SRC_URI could not + # be parsed for one or more ebuilds. There's no point in + # producing a false error here since the root cause will + # produce a valid error elsewhere, such as "SRC_URI.syntax" + # or "ebuild.sytax". + myfiles_all = set(myfiles_all) + for entry in _digests: + if entry not in myfiles_all: + self.qatracker.add_error("digest.unused", checkdir + "::" + entry) + for entry in myfiles_all: + if entry not in _digests: + self.qatracker.add_error("digest.missing", checkdir + "::" + entry) + del myfiles_all + + if os.path.exists(checkdir + "/files"): + filesdirlist = os.listdir(checkdir + "/files") + + # Recurse through files directory, use filesdirlist as a stack; + # appending directories as needed, + # so people can't hide > 20k files in a subdirectory. + while filesdirlist: + y = filesdirlist.pop(0) + relative_path = os.path.join(xpkg, "files", y) + full_path = os.path.join(self.repo_settings.repodir, relative_path) + try: + mystat = os.stat(full_path) + except OSError as oe: + if oe.errno == 2: + # don't worry about it. it likely was removed via fix above. + continue + else: + raise oe + if S_ISDIR(mystat.st_mode): + if self.vcs_settings.status.isVcsDir(y): + continue + for z in os.listdir(checkdir + "/files/" + y): + if self.vcs_settings.status.isVcsDir(z): + continue + filesdirlist.append(y + "/" + z) + # Current policy is no files over 20 KiB, these are the checks. + # File size between 20 KiB and 60 KiB causes a warning, + # while file size over 60 KiB causes an error. + elif mystat.st_size > 61440: + self.qatracker.add_error( + "file.size.fatal", "(%d KiB) %s/files/%s" % ( + mystat.st_size // 1024, xpkg, y)) + elif mystat.st_size > 20480: + self.qatracker.add_error( + "file.size", "(%d KiB) %s/files/%s" % ( + mystat.st_size // 1024, xpkg, y)) + + index = self.repo_settings.repo_config.find_invalid_path_char(y) + if index != -1: + y_relative = os.path.join(checkdir_relative, "files", y) + if self.vcs_settings.vcs is not None \ + and not vcs_new_changed(y_relative, changed, new): + # If the file isn't in the VCS new or changed set, then + # assume that it's an irrelevant temporary file (Manifest + # entries are not generated for file names containing + # prohibited characters). See bug #406877. + index = -1 + if index != -1: + self.qatracker.add_error( + "file.name", + "%s/files/%s: char '%s'" % (checkdir, y, y[index])) + return False + + def digests(self, checkdir): + '''Returns the freshly loaded digests + + @param checkdir: string, directory path + ''' + mf = self.repoman_settings.repositories.get_repo_for_location( + os.path.dirname(os.path.dirname(checkdir))) + mf = mf.load_manifest(checkdir, self.repoman_settings["DISTDIR"]) + _digests = mf.getTypeDigests("DIST") + del mf + return _digests + + def check_mirrors(self, **kwargs): + '''Check that URIs don't reference a server from thirdpartymirrors + + @param ebuild: Ebuild which we check (object). + @returns: boolean + ''' + ebuild = kwargs.get('ebuild').get() + + for uri in portage.dep.use_reduce( + ebuild.metadata["SRC_URI"], matchall=True, is_src_uri=True, + eapi=ebuild.eapi, flat=True): + contains_mirror = False + for mirror, mirror_alias in self.thirdpartymirrors.items(): + if uri.startswith(mirror): + contains_mirror = True + break + if not contains_mirror: + continue + + new_uri = "mirror://%s/%s" % (mirror_alias, uri[len(mirror):]) + self.qatracker.add_error( + "SRC_URI.mirror", + "%s: '%s' found in thirdpartymirrors, use '%s'" % ( + ebuild.relative_path, mirror, new_uri)) + return False + + @property + def runInPkgs(self): + '''Package level scans''' + return (True, [self.check]) + + @property + def runInEbuilds(self): + '''Ebuild level scans''' + return (True, [self.check_mirrors]) diff --git a/repoman/pym/repoman/modules/scan/keywords/__init__.py b/repoman/pym/repoman/modules/scan/keywords/__init__.py new file mode 100644 index 000000000..2223927c8 --- /dev/null +++ b/repoman/pym/repoman/modules/scan/keywords/__init__.py @@ -0,0 +1,33 @@ +# Copyright 2015-2016 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +doc = """Keywords plug-in module for repoman. +Performs keywords checks on ebuilds.""" +__doc__ = doc[:] + + +module_spec = { + 'name': 'keywords', + 'description': doc, + 'provides':{ + 'keywords-module': { + 'name': "keywords", + 'sourcefile': "keywords", + 'class': "KeywordChecks", + 'description': doc, + 'functions': ['prepare', 'check'], + 'func_desc': { + }, + 'mod_kwargs': ['qatracker', 'options', 'repo_metadata', 'profiles', + ], + 'func_kwargs': { + 'changed': (None, None), + 'ebuild': ('Future', 'UNSET'), + 'pkg': ('Future', 'UNSET'), + 'xpkg': None, + 'y_ebuild': (None, None), + }, + }, + } +} + diff --git a/repoman/pym/repoman/modules/scan/keywords/keywords.py b/repoman/pym/repoman/modules/scan/keywords/keywords.py new file mode 100644 index 000000000..7cb2fe912 --- /dev/null +++ b/repoman/pym/repoman/modules/scan/keywords/keywords.py @@ -0,0 +1,133 @@ +# -*- coding:utf-8 -*- + +'''keywords.py +Perform KEYWORDS related checks + +''' + +from repoman.modules.scan.scanbase import ScanBase + + +class KeywordChecks(ScanBase): + '''Perform checks on the KEYWORDS of an ebuild''' + + def __init__(self, **kwargs): + ''' + @param qatracker: QATracker instance + @param options: argparse options instance + ''' + super(KeywordChecks, self).__init__(**kwargs) + self.qatracker = kwargs.get('qatracker') + self.options = kwargs.get('options') + self.repo_metadata = kwargs.get('repo_metadata') + self.profiles = kwargs.get('profiles') + self.slot_keywords = {} + + def prepare(self, **kwargs): + '''Prepare the checks for the next package.''' + self.slot_keywords = {} + return False + + def check(self, **kwargs): + '''Perform the check. + + @param pkg: Package in which we check (object). + @param xpkg: Package in which we check (string). + @param ebuild: Ebuild which we check (object). + @param y_ebuild: Ebuild which we check (string). + @param ebuild_archs: Just the architectures (no prefixes) of the ebuild. + @param changed: Changes instance + @returns: dictionary + ''' + pkg = kwargs.get('pkg').get() + xpkg =kwargs.get('xpkg') + ebuild = kwargs.get('ebuild').get() + y_ebuild = kwargs.get('y_ebuild') + changed = kwargs.get('changed') + if not self.options.straight_to_stable: + self._checkAddedWithStableKeywords( + xpkg, ebuild, y_ebuild, ebuild.keywords, changed) + + self._checkForDroppedKeywords(pkg, ebuild, ebuild.archs) + + self._checkForInvalidKeywords(ebuild, xpkg, y_ebuild) + + self._checkForMaskLikeKeywords(xpkg, y_ebuild, ebuild.keywords) + + self.slot_keywords[pkg.slot].update(ebuild.archs) + return False + + @staticmethod + def _isKeywordStable(keyword): + return not keyword.startswith("~") and not keyword.startswith("-") + + def _checkAddedWithStableKeywords( + self, package, ebuild, y_ebuild, keywords, changed): + catdir, pkgdir = package.split("/") + + stable_keywords = list(filter(self._isKeywordStable, keywords)) + if stable_keywords: + if ebuild.ebuild_path in changed.new_ebuilds and catdir != "virtual": + stable_keywords.sort() + self.qatracker.add_error( + "KEYWORDS.stable", + "%s/%s.ebuild added with stable keywords: %s" % + (package, y_ebuild, " ".join(stable_keywords))) + + def _checkForDroppedKeywords( + self, pkg, ebuild, ebuild_archs): + previous_keywords = self.slot_keywords.get(pkg.slot) + if previous_keywords is None: + self.slot_keywords[pkg.slot] = set() + elif ebuild_archs and "*" not in ebuild_archs and not ebuild.live_ebuild: + dropped_keywords = previous_keywords.difference(ebuild_archs) + if dropped_keywords: + self.qatracker.add_error( + "KEYWORDS.dropped", "%s: %s" % ( + ebuild.relative_path, + " ".join(sorted(dropped_keywords)))) + + def _checkForInvalidKeywords(self, ebuild, xpkg, y_ebuild): + myuse = ebuild.keywords + + for mykey in myuse: + if mykey not in ("-*", "*", "~*"): + myskey = mykey + + if not self._isKeywordStable(myskey[:1]): + myskey = myskey[1:] + + if myskey not in self.repo_metadata['kwlist']: + self.qatracker.add_error("KEYWORDS.invalid", + "%s/%s.ebuild: %s" % (xpkg, y_ebuild, mykey)) + elif myskey not in self.profiles: + self.qatracker.add_error( + "KEYWORDS.invalid", + "%s/%s.ebuild: %s (profile invalid)" + % (xpkg, y_ebuild, mykey)) + + def _checkForMaskLikeKeywords(self, xpkg, y_ebuild, keywords): + # KEYWORDS="-*" is a stupid replacement for package.mask + # and screws general KEYWORDS semantics + if "-*" in keywords: + haskeyword = False + + for kw in keywords: + if kw[0] == "~": + kw = kw[1:] + if kw in self.repo_metadata['kwlist']: + haskeyword = True + + if not haskeyword: + self.qatracker.add_error("KEYWORDS.stupid", + "%s/%s.ebuild" % (xpkg, y_ebuild)) + + @property + def runInPkgs(self): + '''Package level scans''' + return (True, [self.prepare]) + + @property + def runInEbuilds(self): + '''Ebuild level scans''' + return (True, [self.check]) diff --git a/repoman/pym/repoman/modules/scan/manifest/__init__.py b/repoman/pym/repoman/modules/scan/manifest/__init__.py new file mode 100644 index 000000000..dca431b62 --- /dev/null +++ b/repoman/pym/repoman/modules/scan/manifest/__init__.py @@ -0,0 +1,30 @@ +# Copyright 2015-2016 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +doc = """Ebuild plug-in module for repoman. +Performs an IsEbuild check on ebuilds.""" +__doc__ = doc[:] + + +module_spec = { + 'name': 'manifest', + 'description': doc, + 'provides':{ + 'manifest-module': { + 'name': "manifests", + 'sourcefile': "manifests", + 'class': "Manifests", + 'description': doc, + 'functions': ['check', 'create_manifest', 'digest_check'], + 'func_desc': { + }, + 'mod_kwargs': ['options', 'portdb', 'qatracker', 'repo_settings', + ], + 'func_kwargs': { + 'checkdir': (None, None), + 'xpkg': (None, None), + }, + }, + } +} + diff --git a/repoman/pym/repoman/modules/scan/manifest/manifests.py b/repoman/pym/repoman/modules/scan/manifest/manifests.py new file mode 100644 index 000000000..2b8d7af77 --- /dev/null +++ b/repoman/pym/repoman/modules/scan/manifest/manifests.py @@ -0,0 +1,139 @@ +# -*- coding:utf-8 -*- + +import logging +import sys + +# import our initialized portage instance +from repoman._portage import portage +from repoman.modules.scan.scanbase import ScanBase + +from portage import os +from portage.package.ebuild.digestgen import digestgen +from portage.util import writemsg_level + + +class Manifests(ScanBase): + '''Creates as well as checks pkg Manifest entries/files''' + + def __init__(self, **kwargs): + '''Class init + + @param options: the run time cli options + @param portdb: portdb instance + @param qatracker: QATracker instance + @param repo_settings: repository settings instance + ''' + self.options = kwargs.get('options') + self.portdb = kwargs.get('portdb') + self.qatracker = kwargs.get('qatracker') + self.repoman_settings = kwargs.get('repo_settings').repoman_settings + self.generated_manifest = False + + def check(self, **kwargs): + '''Perform a changelog and untracked checks on the ebuild + + @param xpkg: Package in which we check (object). + @param checkdirlist: list of files in the current package directory + @returns: dictionary + ''' + checkdir = kwargs.get('checkdir') + xpkg = kwargs.get('xpkg') + self.generated_manifest = False + self.digest_only = self.options.mode != 'manifest-check' \ + and self.options.digest == 'y' + if self.options.pretend: + return False + if self.options.mode in ("manifest", 'commit', 'fix') or self.digest_only: + failed = False + self.auto_assumed = set() + fetchlist_dict = portage.FetchlistDict( + checkdir, self.repoman_settings, self.portdb) + if self.options.mode == 'manifest' and self.options.force: + portage._doebuild_manifest_exempt_depend += 1 + self.create_manifest(checkdir, fetchlist_dict) + self.repoman_settings["O"] = checkdir + try: + self.generated_manifest = digestgen( + mysettings=self.repoman_settings, myportdb=self.portdb) + except portage.exception.PermissionDenied as e: + self.generated_manifest = False + writemsg_level( + "!!! Permission denied: '%s'\n" % (e,), + level=logging.ERROR, noiselevel=-1) + + if not self.generated_manifest: + writemsg_level( + "Unable to generate manifest.", + level=logging.ERROR, noiselevel=-1) + failed = True + + if self.options.mode == "manifest": + if not failed and self.options.force and self.auto_assumed and \ + 'assume-digests' in self.repoman_settings.features: + # Show which digests were assumed despite the --force option + # being given. This output will already have been shown by + # digestgen() if assume-digests is not enabled, so only show + # it here if assume-digests is enabled. + pkgs = list(fetchlist_dict) + pkgs.sort() + portage.writemsg_stdout( + " digest.assumed %s" % + portage.output.colorize( + "WARN", str(len(self.auto_assumed)).rjust(18)) + "\n") + for cpv in pkgs: + fetchmap = fetchlist_dict[cpv] + pf = portage.catsplit(cpv)[1] + for distfile in sorted(fetchmap): + if distfile in self.auto_assumed: + portage.writemsg_stdout( + " %s::%s\n" % (pf, distfile)) + # continue, skip remaining main loop code + return True + elif failed: + sys.exit(1) + if not self.generated_manifest: + self.digest_check(xpkg, checkdir) + if self.options.mode == 'manifest-check': + return True + return False + + def create_manifest(self, checkdir, fetchlist_dict): + '''Creates a Manifest file + + @param checkdir: the directory to generate the Manifest in + @param fetchlist_dict: dictionary of files to fetch and/or include + in the manifest + ''' + try: + distdir = self.repoman_settings['DISTDIR'] + mf = self.repoman_settings.repositories.get_repo_for_location( + os.path.dirname(os.path.dirname(checkdir))) + mf = mf.load_manifest( + checkdir, distdir, fetchlist_dict=fetchlist_dict) + mf.create( + requiredDistfiles=None, assumeDistHashesAlways=True) + for distfiles in fetchlist_dict.values(): + for distfile in distfiles: + if os.path.isfile(os.path.join(distdir, distfile)): + mf.fhashdict['DIST'].pop(distfile, None) + else: + self.auto_assumed.add(distfile) + mf.write() + finally: + portage._doebuild_manifest_exempt_depend -= 1 + + def digest_check(self, xpkg, checkdir): + '''Check the manifest entries, report any Q/A errors + + @param xpkg: the cat/pkg name to check + @param checkdir: the directory path to check''' + self.repoman_settings['O'] = checkdir + self.repoman_settings['PORTAGE_QUIET'] = '1' + if not portage.digestcheck([], self.repoman_settings, strict=1): + self.qatracker.add_error("manifest.bad", os.path.join(xpkg, 'Manifest')) + self.repoman_settings.pop('PORTAGE_QUIET', None) + + @property + def runInPkgs(self): + '''Package level scans''' + return (True, [self.check]) diff --git a/repoman/pym/repoman/modules/scan/metadata/__init__.py b/repoman/pym/repoman/modules/scan/metadata/__init__.py new file mode 100644 index 000000000..b656d7af0 --- /dev/null +++ b/repoman/pym/repoman/modules/scan/metadata/__init__.py @@ -0,0 +1,85 @@ +# Copyright 2015-2016 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +doc = """Metadata plug-in module for repoman. +Performs metadata checks on packages.""" +__doc__ = doc[:] + + +module_spec = { + 'name': 'metadata', + 'description': doc, + 'provides':{ + 'pkg-metadata': { + 'name': "pkgmetadata", + 'sourcefile': "pkgmetadata", + 'class': "PkgMetadata", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + 'mod_kwargs': ['repo_settings', 'qatracker', 'options', + 'metadata_xsd', 'uselist', + ], + 'func_kwargs': { + 'checkdir': (None, None), + 'checkdirlist': (None, None), + 'ebuild': (None, None), + 'pkg': (None, None), + 'repolevel': (None, None), + 'validity_future': (None, None), + 'xpkg': (None, None), + 'y_ebuild': (None, None), + }, + }, + 'ebuild-metadata': { + 'name': "ebuild_metadata", + 'sourcefile': "ebuild_metadata", + 'class': "EbuildMetadata", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + 'mod_kwargs': ['qatracker', + ], + 'func_kwargs': { + 'catdir': (None, None), + 'ebuild': (None, None), + 'xpkg': (None, None), + 'y_ebuild': (None, None), + }, + }, + 'description-metadata': { + 'name': "description", + 'sourcefile': "description", + 'class': "DescriptionChecks", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + 'mod_kwargs': ['qatracker', + ], + 'func_kwargs': { + 'ebuild': (None, None), + 'pkg': ('Future', 'UNSET'), + }, + }, + 'restrict-metadata': { + 'name': "restrict", + 'sourcefile': "restrict", + 'class': "RestrictChecks", + 'description': doc, + 'functions': ['check'], + 'func_desc': { + }, + 'mod_kwargs': ['qatracker', + ], + 'func_kwargs': { + 'ebuild': (None, None), + 'xpkg': (None, None), + 'y_ebuild': (None, None), + }, + }, + } +} + diff --git a/repoman/pym/repoman/modules/scan/metadata/description.py b/repoman/pym/repoman/modules/scan/metadata/description.py new file mode 100644 index 000000000..79f62e1de --- /dev/null +++ b/repoman/pym/repoman/modules/scan/metadata/description.py @@ -0,0 +1,41 @@ + +'''description.py +Perform checks on the DESCRIPTION variable. +''' + +from repoman.modules.scan.scanbase import ScanBase +from repoman.qa_data import max_desc_len + + +class DescriptionChecks(ScanBase): + '''Perform checks on the DESCRIPTION variable.''' + + def __init__(self, **kwargs): + ''' + @param qatracker: QATracker instance + ''' + self.qatracker = kwargs.get('qatracker') + + def checkTooLong(self, **kwargs): + ''' + @param pkg: Package in which we check (object). + @param ebuild: Ebuild which we check (object). + ''' + ebuild = kwargs.get('ebuild').get() + pkg = kwargs.get('pkg').get() + # 14 is the length of DESCRIPTION="" + if len(pkg._metadata['DESCRIPTION']) > max_desc_len: + self.qatracker.add_error( + 'DESCRIPTION.toolong', + "%s: DESCRIPTION is %d characters (max %d)" % + (ebuild.relative_path, len( + pkg._metadata['DESCRIPTION']), max_desc_len)) + return False + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (True, [self.checkTooLong]) diff --git a/repoman/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/repoman/pym/repoman/modules/scan/metadata/ebuild_metadata.py new file mode 100644 index 000000000..e991a30b3 --- /dev/null +++ b/repoman/pym/repoman/modules/scan/metadata/ebuild_metadata.py @@ -0,0 +1,71 @@ +# -*- coding:utf-8 -*- + +'''Ebuild Metadata Checks''' + +import re +import sys + +if sys.hexversion >= 0x3000000: + basestring = str + +from repoman.modules.scan.scanbase import ScanBase +from repoman.qa_data import missingvars + +NON_ASCII_RE = re.compile(r'[^\x00-\x7f]') + + +class EbuildMetadata(ScanBase): + + def __init__(self, **kwargs): + self.qatracker = kwargs.get('qatracker') + + def invalidchar(self, **kwargs): + ebuild = kwargs.get('ebuild').get() + for k, v in ebuild.metadata.items(): + if not isinstance(v, basestring): + continue + m = NON_ASCII_RE.search(v) + if m is not None: + self.qatracker.add_error( + "variable.invalidchar", + "%s: %s variable contains non-ASCII " + "character at position %s" % + (ebuild.relative_path, k, m.start() + 1)) + return False + + def missing(self, **kwargs): + ebuild = kwargs.get('ebuild').get() + for pos, missing_var in enumerate(missingvars): + if not ebuild.metadata.get(missing_var): + if kwargs.get('catdir') == "virtual" and \ + missing_var in ("HOMEPAGE", "LICENSE"): + continue + if ebuild.live_ebuild and missing_var == "KEYWORDS": + continue + myqakey = missingvars[pos] + ".missing" + self.qatracker.add_error(myqakey, '%s/%s.ebuild' + % (kwargs.get('xpkg'), kwargs.get('y_ebuild'))) + return False + + def old_virtual(self, **kwargs): + ebuild = kwargs.get('ebuild').get() + if ebuild.metadata.get("PROVIDE"): + self.qatracker.add_error("virtual.oldstyle", ebuild.relative_path) + return False + + def virtual(self, **kwargs): + ebuild = kwargs.get('ebuild').get() + if kwargs.get('catdir') == "virtual": + for var in ("HOMEPAGE", "LICENSE"): + if ebuild.metadata.get(var): + myqakey = var + ".virtual" + self.qatracker.add_error(myqakey, ebuild.relative_path) + return False + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (True, [self.invalidchar, self.missing, self.old_virtual, self.virtual]) diff --git a/repoman/pym/repoman/modules/scan/metadata/pkgmetadata.py b/repoman/pym/repoman/modules/scan/metadata/pkgmetadata.py new file mode 100644 index 000000000..433551aed --- /dev/null +++ b/repoman/pym/repoman/modules/scan/metadata/pkgmetadata.py @@ -0,0 +1,247 @@ +# -*- coding:utf-8 -*- + +'''Package Metadata Checks operations''' + +import sys + +from itertools import chain + +try: + from lxml import etree + from lxml.etree import ParserError +except (SystemExit, KeyboardInterrupt): + raise +except (ImportError, SystemError, RuntimeError, Exception): + # broken or missing xml support + # http://bugs.python.org/issue14988 + msg = ["Please emerge dev-python/lxml in order to use repoman."] + from portage.output import EOutput + out = EOutput() + for line in msg: + out.eerror(line) + sys.exit(1) + +# import our initialized portage instance +from repoman._portage import portage +from repoman.metadata import metadata_dtd_uri +from repoman.modules.scan.scanbase import ScanBase + +from portage.exception import InvalidAtom +from portage import os +from portage.dep import Atom + +from .use_flags import USEFlagChecks + +if sys.hexversion >= 0x3000000: + # pylint: disable=W0622 + basestring = str + +metadata_xml_encoding = 'UTF-8' +metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' \ + % (metadata_xml_encoding,) +metadata_doctype_name = 'pkgmetadata' + + +class PkgMetadata(ScanBase, USEFlagChecks): + '''Package metadata.xml checks''' + + def __init__(self, **kwargs): + '''PkgMetadata init function + + @param repo_settings: settings instance + @param qatracker: QATracker instance + @param options: argparse options instance + @param metadata_xsd: path of metadata.xsd + ''' + super(PkgMetadata, self).__init__(**kwargs) + repo_settings = kwargs.get('repo_settings') + self.qatracker = kwargs.get('qatracker') + self.options = kwargs.get('options') + self.metadata_xsd = kwargs.get('metadata_xsd') + self.globalUseFlags = kwargs.get('uselist') + self.repoman_settings = repo_settings.repoman_settings + self.musedict = {} + self.muselist = set() + + def check(self, **kwargs): + '''Performs the checks on the metadata.xml for the package + @param xpkg: the pacakge being checked + @param checkdir: string, directory path + @param checkdirlist: list of checkdir's + @param repolevel: integer + @returns: boolean + ''' + xpkg = kwargs.get('xpkg') + checkdir = kwargs.get('checkdir') + checkdirlist = kwargs.get('checkdirlist').get() + + self.musedict = {} + if self.options.mode in ['manifest']: + self.muselist = frozenset(self.musedict) + return False + + # metadata.xml file check + if "metadata.xml" not in checkdirlist: + self.qatracker.add_error("metadata.missing", xpkg + "/metadata.xml") + self.muselist = frozenset(self.musedict) + return False + + # metadata.xml parse check + metadata_bad = False + + # read metadata.xml into memory + try: + _metadata_xml = etree.parse(os.path.join(checkdir, 'metadata.xml')) + except (ParserError, SyntaxError, EnvironmentError) as e: + metadata_bad = True + self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e)) + del e + self.muselist = frozenset(self.musedict) + return False + + xml_encoding = _metadata_xml.docinfo.encoding + if xml_encoding.upper() != metadata_xml_encoding: + self.qatracker.add_error( + "metadata.bad", "%s/metadata.xml: " + "xml declaration encoding should be '%s', not '%s'" % + (xpkg, metadata_xml_encoding, xml_encoding)) + + if not _metadata_xml.docinfo.doctype: + metadata_bad = True + self.qatracker.add_error( + "metadata.bad", + "%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing")) + else: + doctype_system = _metadata_xml.docinfo.system_url + if doctype_system != metadata_dtd_uri: + if doctype_system is None: + system_problem = "but it is undefined" + else: + system_problem = "not '%s'" % doctype_system + self.qatracker.add_error( + "metadata.bad", "%s/metadata.xml: " + "DOCTYPE: SYSTEM should refer to '%s', %s" % + (xpkg, metadata_dtd_uri, system_problem)) + doctype_name = _metadata_xml.docinfo.doctype.split(' ')[1] + if doctype_name != metadata_doctype_name: + self.qatracker.add_error( + "metadata.bad", "%s/metadata.xml: " + "DOCTYPE: name should be '%s', not '%s'" % + (xpkg, metadata_doctype_name, doctype_name)) + + # load USE flags from metadata.xml + self.musedict = self._parse_metadata_use(_metadata_xml, xpkg) + for atom in chain(*self.musedict.values()): + if atom is None: + continue + try: + atom = Atom(atom) + except InvalidAtom as e: + self.qatracker.add_error( + "metadata.bad", + "%s/metadata.xml: Invalid atom: %s" % (xpkg, e)) + else: + if atom.cp != xpkg: + self.qatracker.add_error( + "metadata.bad", + "%s/metadata.xml: Atom contains " + "unexpected cat/pn: %s" % (xpkg, atom)) + + # Only carry out if in package directory or check forced + if not metadata_bad: + validator = etree.XMLSchema(file=self.metadata_xsd) + if not validator.validate(_metadata_xml): + self._add_validate_errors(xpkg, validator.error_log) + self.muselist = frozenset(self.musedict) + return False + + def check_unused(self, **kwargs): + '''Reports on any unused metadata.xml use descriptions + + @param xpkg: the pacakge being checked + @param used_useflags: use flag list + @param validity_future: Future instance + ''' + xpkg = kwargs.get('xpkg') + valid_state = kwargs.get('validity_future').get() + # check if there are unused local USE-descriptions in metadata.xml + # (unless there are any invalids, to avoid noise) + if valid_state: + for myflag in self.muselist.difference(self.usedUseFlags): + self.qatracker.add_error( + "metadata.warning", + "%s/metadata.xml: unused local USE-description: '%s'" + % (xpkg, myflag)) + return False + + def _parse_metadata_use(self, xml_tree, xpkg): + """ + Records are wrapped in XML as per GLEP 56 + returns a dict with keys constisting of USE flag names and values + containing their respective descriptions + """ + uselist = {} + + usetags = xml_tree.findall("use") + if not usetags: + return uselist + + # It's possible to have multiple 'use' elements. + for usetag in usetags: + flags = usetag.findall("flag") + if not flags: + # DTD allows use elements containing no flag elements. + continue + + for flag in flags: + pkg_flag = flag.get("name") + if pkg_flag is not None: + flag_restrict = flag.get("restrict") + + # emulate the Element.itertext() method from python-2.7 + inner_text = [] + stack = [] + stack.append(flag) + while stack: + obj = stack.pop() + if isinstance(obj, basestring): + inner_text.append(obj) + continue + if isinstance(obj.text, basestring): + inner_text.append(obj.text) + if isinstance(obj.tail, basestring): + stack.append(obj.tail) + stack.extend(reversed(obj)) + + if flag.get("name") not in uselist: + uselist[flag.get("name")] = {} + + # (flag_restrict can be None) + uselist[flag.get("name")][flag_restrict] = " ".join("".join(inner_text).split()) + return uselist + + def _add_validate_errors(self, xpkg, log): + listed = set() + for error in log: + msg_prefix = error.message.split(":",1)[0] + info = "%s %s" % (error.line, msg_prefix) + if info not in listed: + listed.add(info) + self.qatracker.add_error( + "metadata.bad", + "%s/metadata.xml: line: %s, %s" + % (xpkg, error.line, error.message)) + + @property + def runInPkgs(self): + '''Package level scans''' + return (True, [self.check]) + + @property + def runInEbuilds(self): + return (True, [self.check_useflags]) + + @property + def runInFinal(self): + '''Final scans at the package level''' + return (True, [self.check_unused]) diff --git a/repoman/pym/repoman/modules/scan/metadata/restrict.py b/repoman/pym/repoman/modules/scan/metadata/restrict.py new file mode 100644 index 000000000..0f9c5e52e --- /dev/null +++ b/repoman/pym/repoman/modules/scan/metadata/restrict.py @@ -0,0 +1,53 @@ + +'''restrict.py +Perform checks on the RESTRICT variable. +''' + +# import our initialized portage instance +from repoman._portage import portage + +from repoman.modules.scan.scanbase import ScanBase +from repoman.qa_data import valid_restrict + + +class RestrictChecks(ScanBase): + '''Perform checks on the RESTRICT variable.''' + + def __init__(self, **kwargs): + ''' + @param qatracker: QATracker instance + ''' + self.qatracker = kwargs.get('qatracker') + + def check(self, **kwargs): + xpkg = kwargs.get('xpkg') + ebuild = kwargs.get('ebuild').get() + y_ebuild = kwargs.get('y_ebuild') + myrestrict = None + + try: + myrestrict = portage.dep.use_reduce( + ebuild.metadata["RESTRICT"], matchall=1, flat=True) + except portage.exception.InvalidDependString as e: + self.qatracker.add_error("RESTRICT.syntax", + "%s: RESTRICT: %s" % (ebuild.relative_path, e)) + del e + + if myrestrict: + myrestrict = set(myrestrict) + mybadrestrict = myrestrict.difference(valid_restrict) + + if mybadrestrict: + for mybad in mybadrestrict: + self.qatracker.add_error("RESTRICT.invalid", + "%s/%s.ebuild: %s" % (xpkg, y_ebuild, mybad)) + return False + + @property + def runInPkgs(self): + return (False, []) + + @property + def runInEbuilds(self): + return (True, [self.check]) + diff --git a/repoman/pym/repoman/modules/scan/metadata/use_flags.py b/repoman/pym/repoman/modules/scan/metadata/use_flags.py new file mode 100644 index 000000000..1738fd23e --- /dev/null +++ b/repoman/pym/repoman/modules/scan/metadata/use_flags.py @@ -0,0 +1,94 @@ +# -*- coding:utf-8 -*- + +'''use_flags.py +Performs USE flag related checks +''' + +# import our centrally initialized portage instance +from repoman._portage import portage + +from portage import eapi +from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use + + +class USEFlagChecks(object): + '''Performs checks on USE flags listed in the ebuilds and metadata.xml''' + + def __init__(self, **kwargs): + '''Class init + + @param qatracker: QATracker instance + @param globalUseFlags: Global USE flags + ''' + super(USEFlagChecks, self).__init__() + self.qatracker = None + self.globalUseFlags = None + self.useFlags = [] + self.defaultUseFlags = [] + self.usedUseFlags = set() + + def check_useflags(self, **kwargs): + '''Perform the check. + + @param pkg: Package in which we check (object). + @param xpkg: Package in which we check (string). + @param ebuild: Ebuild which we check (object). + @param y_ebuild: Ebuild which we check (string). + @returns: dictionary, including {ebuild_UsedUseFlags, used_useflags} + ''' + pkg = kwargs.get('pkg').get() + package = kwargs.get('xpkg') + ebuild = kwargs.get('ebuild').get() + y_ebuild = kwargs.get('y_ebuild') + # reset state variables for the run + self.useFlags = [] + self.defaultUseFlags = [] + # perform the checks + self._checkGlobal(pkg) + self._checkMetadata(package, ebuild, y_ebuild, self.muselist) + self._checkRequiredUSE(pkg, ebuild) + return False + + + def _checkGlobal(self, pkg): + for myflag in pkg._metadata["IUSE"].split(): + flag_name = myflag.lstrip("+-") + self.usedUseFlags.add(flag_name) + if myflag != flag_name: + self.defaultUseFlags.append(myflag) + if flag_name not in self.globalUseFlags: + self.useFlags.append(flag_name) + + def _checkMetadata(self, package, ebuild, y_ebuild, localUseFlags): + for mypos in range(len(self.useFlags) - 1, -1, -1): + if self.useFlags[mypos] and (self.useFlags[mypos] in localUseFlags): + del self.useFlags[mypos] + + if self.defaultUseFlags and not eapi_has_iuse_defaults(eapi): + for myflag in self.defaultUseFlags: + self.qatracker.add_error( + 'EAPI.incompatible', "%s: IUSE defaults" + " not supported with EAPI='%s': '%s'" % ( + ebuild.relative_path, eapi, myflag)) + + for mypos in range(len(self.useFlags)): + self.qatracker.add_error( + "IUSE.invalid", + "%s/%s.ebuild: %s" % (package, y_ebuild, self.useFlags[mypos])) + + def _checkRequiredUSE(self, pkg, ebuild): + required_use = pkg._metadata["REQUIRED_USE"] + if required_use: + if not eapi_has_required_use(eapi): + self.qatracker.add_error( + 'EAPI.incompatible', "%s: REQUIRED_USE" + " not supported with EAPI='%s'" + % (ebuild.relative_path, eapi,)) + try: + portage.dep.check_required_use( + required_use, (), pkg.iuse.is_valid_flag, eapi=eapi) + except portage.exception.InvalidDependString as e: + self.qatracker.add_error( + "REQUIRED_USE.syntax", + "%s: REQUIRED_USE: %s" % (ebuild.relative_path, e)) + del e diff --git a/repoman/pym/repoman/modules/scan/options/__init__.py b/repoman/pym/repoman/modules/scan/options/__init__.py new file mode 100644 index 000000000..a5746ce67 --- /dev/null +++ b/repoman/pym/repoman/modules/scan/options/__init__.py @@ -0,0 +1,28 @@ +# Copyright 2015-2016 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +doc = """Options plug-in module for repoman. +Performs option related actions on ebuilds.""" +__doc__ = doc[:] + + +module_spec = { + 'name': 'options', + 'description': doc, + 'provides':{ + 'options-module': { + 'name': "options", + 'sourcefile': "options", + 'class': "Options", + 'description': doc, + 'functions': ['is_forced'], + 'func_desc': { + }, + 'mod_kwargs': ['options', + ], + 'func_kwargs': { + }, + }, + } +} + diff --git a/repoman/pym/repoman/modules/scan/options/options.py b/repoman/pym/repoman/modules/scan/options/options.py new file mode 100644 index 000000000..443f01bd8 --- /dev/null +++ b/repoman/pym/repoman/modules/scan/options/options.py @@ -0,0 +1,29 @@ + +from repoman.modules.scan.scanbase import ScanBase + + +class Options(ScanBase): + + def __init__(self, **kwargs): + '''Class init function + + @param options: argparse options instance + ''' + self.options = kwargs.get('options') + + def is_forced(self, **kwargs): + '''Simple boolean function to trigger a skip past some additional checks + + @returns: dictionary + ''' + if self.options.force: + # The dep_check() calls are the most expensive QA test. If --force + # is enabled, there's no point in wasting time on these since the + # user is intent on forcing the commit anyway. + return True + return False + + @property + def runInEbuilds(self): + '''Ebuild level scans''' + return (True, [self.is_forced]) diff --git a/repoman/pym/repoman/modules/scan/scan.py b/repoman/pym/repoman/modules/scan/scan.py new file mode 100644 index 000000000..d2a5f515b --- /dev/null +++ b/repoman/pym/repoman/modules/scan/scan.py @@ -0,0 +1,66 @@ +# -*- coding:utf-8 -*- + +''' +moudules/scan.py +Module specific package scan list generator +''' + +import logging +import os +import sys + +from repoman.errors import caterror + + +def scan(repolevel, reposplit, startdir, categories, repo_settings): + '''Generate a list of pkgs to scan + + @param repolevel: integer, number of subdirectories deep from the tree root + @param reposplit: list of the path subdirs + @param startdir: the top level directory to begin scanning from + @param categories: list of known categories + @param repo_settings: repository settings instance + @returns: scanlist, sorted list of pkgs to scan + ''' + scanlist = [] + if repolevel == 2: + # we are inside a category directory + catdir = reposplit[-1] + if catdir not in categories: + caterror(catdir, repo_settings.repodir) + mydirlist = os.listdir(startdir) + for x in mydirlist: + if x == "CVS" or x.startswith("."): + continue + if os.path.isdir(startdir + "/" + x): + scanlist.append(catdir + "/" + x) + # repo_subdir = catdir + os.sep + elif repolevel == 1: + for x in categories: + if not os.path.isdir(startdir + "/" + x): + continue + for y in os.listdir(startdir + "/" + x): + if y == "CVS" or y.startswith("."): + continue + if os.path.isdir(startdir + "/" + x + "/" + y): + scanlist.append(x + "/" + y) + # repo_subdir = "" + elif repolevel == 3: + catdir = reposplit[-2] + if catdir not in categories: + caterror(catdir, repo_settings.repodir) + scanlist.append(catdir + "/" + reposplit[-1]) + # repo_subdir = scanlist[-1] + os.sep + else: + msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \ + ' from the current working directory' + logging.critical(msg) + sys.exit(1) + + # repo_subdir_len = len(repo_subdir) + scanlist.sort() + + logging.debug( + "Found the following packages to scan:\n%s" % '\n'.join(scanlist)) + + return scanlist diff --git a/repoman/pym/repoman/modules/scan/scanbase.py b/repoman/pym/repoman/modules/scan/scanbase.py new file mode 100644 index 000000000..aea1bb121 --- /dev/null +++ b/repoman/pym/repoman/modules/scan/scanbase.py @@ -0,0 +1,79 @@ +# -*- coding:utf-8 -*- + + +class ScanBase(object): + '''Skeleton class for performing a scan for one or more items + to check in a pkg directory or ebuild.''' + + def __init__(self, **kwargs): + '''Class init + + @param kwargs: an optional dictionary of common repository + wide parameters that may be required. + ''' + # Since no two checks are identicle as to what kwargs are needed, + # this does not define any from it here. + super(ScanBase, self).__init__() + + """ # sample check + def check_foo(self, **kwargs): + '''Class check skeleton function. Define this for a + specific check to perform. + + @param kwargs: an optional dictionary of dynamic package and or ebuild + specific data that may be required. Dynamic data can + vary depending what checks have run before it. + So execution order can be important. + ''' + # Insert the code for the check here + # It should return a dictionary of at least {'continue': False} + # The continue attribute will default to False if not returned. + # This will allow the loop to continue with the next check in the list. + # Include any additional dynamic data that needs to be added or updated. + return False # used as a continue True/False value + """ + + @property + def runInPkgs(self): + '''Package level scans''' + # default no run (False) and empty list of functions to run + # override this method to define a function or + # functions to run in this process loop + # return a tuple of a boolean or boolean result and an ordered list + # of functions to run. ie: return (True, [self.check_foo]) + # in this way, it can be dynamically determined at run time, if + # later stage scans are to be run. + # This class instance is maintaned for all stages, so data can be + # carried over from stage to stage + # next stage is runInEbuilds + return (False, []) + + @property + def runInEbuilds(self): + '''Ebuild level scans''' + # default empty list of functions to run + # override this method to define a function or + # functions to run in this process loop + # return a tuple of a boolean or boolean result and an ordered list + # of functions to run. ie: return (True, [self.check_bar]) + # in this way, it can be dynamically determined at run time, if + # later stage scans are to be run. + # This class instance is maintaned for all stages, so data can be + # carried over from stage to stage + # next stage is runInFinal + return (False, []) + + @property + def runInFinal(self): + '''Final scans at the package level''' + # default empty list of functions to run + # override this method to define a function or + # functions to run in this process loop + # return a tuple of a boolean or boolean result and an ordered list + # of functions to run. ie: return (True, [self.check_baz]) + # in this way, it can be dynamically determined at run time, if + # later stage scans are to be run. + # This class instance is maintaned for all stages, so data can be + # carried over from stage to stage + # runInFinal is currently the last stage of scans performed. + return (False, []) |