diff options
author | Michał Górny <mgorny@gentoo.org> | 2018-07-17 21:50:45 +0200 |
---|---|---|
committer | Zac Medico <zmedico@gentoo.org> | 2018-07-18 16:19:11 -0700 |
commit | bc0fa8d3795ed7e40aaa00f579bb2977897bce25 (patch) | |
tree | 2a62c721ee8dec47ddb564254e1cbd967577d1f0 /repoman/pym/repoman | |
parent | EventLoop: raise TypeError for unexpected call_* keyword args (diff) | |
download | portage-bc0fa8d3795ed7e40aaa00f579bb2977897bce25.tar.gz portage-bc0fa8d3795ed7e40aaa00f579bb2977897bce25.tar.bz2 portage-bc0fa8d3795ed7e40aaa00f579bb2977897bce25.zip |
Rename pym→lib, for better distutils-r1 interoperability
Closes: https://github.com/gentoo/portage/pull/343
Diffstat (limited to 'repoman/pym/repoman')
139 files changed, 0 insertions, 11005 deletions
diff --git a/repoman/pym/repoman/__init__.py b/repoman/pym/repoman/__init__.py deleted file mode 100644 index 89779b95c..000000000 --- a/repoman/pym/repoman/__init__.py +++ /dev/null @@ -1,79 +0,0 @@ - -import os.path -import subprocess -import sys -import time - -try: - import portage.const - import portage.proxy as proxy - from portage import _encodings, _shell_quote, _unicode_encode, _unicode_decode - from portage.const import PORTAGE_BASE_PATH, BASH_BINARY -except ImportError as e: - sys.stderr.write("\n\n") - sys.stderr.write("!!! Failed to complete portage imports. There are internal modules for\n") - sys.stderr.write("!!! portage and failure here indicates that you have a problem with your\n") - sys.stderr.write("!!! installation of portage. Please try a rescue portage located in the\n") - sys.stderr.write("!!! portage tree under '/usr/portage/sys-apps/portage/files/' (default).\n") - sys.stderr.write("!!! There is a README.RESCUE file that details the steps required to perform\n") - sys.stderr.write("!!! a recovery of portage.\n") - sys.stderr.write(" "+str(e)+"\n\n") - raise - -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - long = int - -VERSION = "HEAD" - -REPOMAN_BASE_PATH = os.path.join(os.sep, os.sep.join(os.path.realpath(__file__.rstrip("co")).split(os.sep)[:-3])) - -_not_installed = os.path.isfile(os.path.join(REPOMAN_BASE_PATH, ".repoman_not_installed")) - -if VERSION == 'HEAD': - class _LazyVersion(proxy.objectproxy.ObjectProxy): - def _get_target(self): - global VERSION - if VERSION is not self: - return VERSION - if os.path.isdir(os.path.join(PORTAGE_BASE_PATH, '.git')): - encoding = _encodings['fs'] - cmd = [BASH_BINARY, "-c", ("cd %s ; git describe --match 'repoman-*' || exit $? ; " + \ - "if [ -n \"`git diff-index --name-only --diff-filter=M HEAD`\" ] ; " + \ - "then echo modified ; git rev-list --format=%%ct -n 1 HEAD ; fi ; " + \ - "exit 0") % _shell_quote(PORTAGE_BASE_PATH)] - cmd = [_unicode_encode(x, encoding=encoding, errors='strict') - for x in cmd] - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - output = _unicode_decode(proc.communicate()[0], encoding=encoding) - status = proc.wait() - if os.WIFEXITED(status) and os.WEXITSTATUS(status) == os.EX_OK: - output_lines = output.splitlines() - if output_lines: - version_split = output_lines[0].split('-') - if len(version_split) > 1: - VERSION = version_split[1] - patchlevel = False - if len(version_split) > 2: - patchlevel = True - VERSION = "%s_p%s" % (VERSION, version_split[2]) - if len(output_lines) > 1 and output_lines[1] == 'modified': - head_timestamp = None - if len(output_lines) > 3: - try: - head_timestamp = long(output_lines[3]) - except ValueError: - pass - timestamp = long(time.time()) - if head_timestamp is not None and timestamp > head_timestamp: - timestamp = timestamp - head_timestamp - if not patchlevel: - VERSION = "%s_p0" % (VERSION,) - VERSION = "%s_p%d" % (VERSION, timestamp) - return VERSION - else: - print("NO output lines :(") - VERSION = 'HEAD' - return VERSION - VERSION = _LazyVersion() diff --git a/repoman/pym/repoman/_portage.py b/repoman/pym/repoman/_portage.py deleted file mode 100644 index 0f611f761..000000000 --- a/repoman/pym/repoman/_portage.py +++ /dev/null @@ -1,25 +0,0 @@ - -'''repoman/_portage.py -Central location for the portage import. -There were problems when portage was imported by submodules -due to the portage instance was somehow different that the -initial portage import in main.py. The later portage imports -did not contain the repo it was working on. That repo was my cvs tree -and not listed in those subsequent portage imports. - -All modules should import portage from this one - -from repoman._portage import portage - -Then continue to import the remaining portage modules needed -''' - -import sys - -from os import path as osp -pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__)))) -sys.path.insert(0, pym_path) - -import portage -portage._internal_caller = True -portage._disable_legacy_globals() diff --git a/repoman/pym/repoman/_subprocess.py b/repoman/pym/repoman/_subprocess.py deleted file mode 100644 index b6c19bda3..000000000 --- a/repoman/pym/repoman/_subprocess.py +++ /dev/null @@ -1,83 +0,0 @@ -# -*- coding:utf-8 -*- - - -import codecs -import subprocess -import sys - -# import our initialized portage instance -from repoman._portage import portage - -from portage import os -from portage.process import find_binary -from portage import _encodings, _unicode_encode - - -def repoman_getstatusoutput(cmd): - """ - Implements an interface similar to getstatusoutput(), but with - customized unicode handling (see bug #310789) and without the shell. - """ - args = portage.util.shlex_split(cmd) - - if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \ - not os.path.isabs(args[0]): - # Python 3.1 _execvp throws TypeError for non-absolute executable - # path passed as bytes (see https://bugs.python.org/issue8513). - fullname = find_binary(args[0]) - if fullname is None: - raise portage.exception.CommandNotFound(args[0]) - args[0] = fullname - - encoding = _encodings['fs'] - args = [ - _unicode_encode(x, encoding=encoding, errors='strict') for x in args] - proc = subprocess.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - output = portage._unicode_decode( - proc.communicate()[0], encoding=encoding, errors='strict') - if output and output[-1] == "\n": - # getstatusoutput strips one newline - output = output[:-1] - return (proc.wait(), output) - - -class repoman_popen(portage.proxy.objectproxy.ObjectProxy): - """ - Implements an interface similar to os.popen(), but with customized - unicode handling (see bug #310789) and without the shell. - """ - - __slots__ = ('_proc', '_stdout') - - def __init__(self, cmd): - args = portage.util.shlex_split(cmd) - - if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \ - not os.path.isabs(args[0]): - # Python 3.1 _execvp throws TypeError for non-absolute executable - # path passed as bytes (see https://bugs.python.org/issue8513). - fullname = find_binary(args[0]) - if fullname is None: - raise portage.exception.CommandNotFound(args[0]) - args[0] = fullname - - encoding = _encodings['fs'] - args = [ - _unicode_encode(x, encoding=encoding, errors='strict') - for x in args] - proc = subprocess.Popen(args, stdout=subprocess.PIPE) - object.__setattr__( - self, '_proc', proc) - object.__setattr__( - self, '_stdout', codecs.getreader(encoding)(proc.stdout, 'strict')) - - def _get_target(self): - return object.__getattribute__(self, '_stdout') - - __enter__ = _get_target - - def __exit__(self, exc_type, exc_value, traceback): - proc = object.__getattribute__(self, '_proc') - proc.wait() - proc.stdout.close() diff --git a/repoman/pym/repoman/actions.py b/repoman/pym/repoman/actions.py deleted file mode 100644 index 8e23322c8..000000000 --- a/repoman/pym/repoman/actions.py +++ /dev/null @@ -1,677 +0,0 @@ -# -*- coding:utf-8 -*- - -from __future__ import print_function, unicode_literals - -import errno -import io -import logging -import platform -import re -import shutil -import signal -import sys -import tempfile -import time -from itertools import chain - -try: - from urllib.parse import parse_qs, urlsplit, urlunsplit -except ImportError: - from urlparse import parse_qs, urlsplit, urlunsplit - -from _emerge.UserQuery import UserQuery - -from repoman._portage import portage -from portage import os -from portage import _encodings -from portage import _unicode_encode -from portage.output import ( - bold, create_color_func, green, red) -from portage.package.ebuild.digestgen import digestgen -from portage.util import writemsg_level - -from repoman.copyrights import update_copyright -from repoman.gpg import gpgsign, need_signature -from repoman import utilities -from repoman.modules.vcs.vcs import vcs_files_to_cps -from repoman import VERSION - -bad = create_color_func("BAD") - - -class Actions(object): - '''Handles post check result output and performs - the various vcs activities for committing the results''' - - def __init__(self, repo_settings, options, scanner, vcs_settings): - self.repo_settings = repo_settings - self.options = options - self.scanner = scanner - self.vcs_settings = vcs_settings - self.repoman_settings = repo_settings.repoman_settings - self.suggest = { - 'ignore_masked': False, - 'include_dev': False, - } - if scanner.have['pmasked'] and not (options.without_mask or options.ignore_masked): - self.suggest['ignore_masked'] = True - if scanner.have['dev_keywords'] and not options.include_dev: - self.suggest['include_dev'] = True - - - def inform(self, can_force, result): - '''Inform the user of all the problems found''' - if ((self.suggest['ignore_masked'] or self.suggest['include_dev']) - and not self.options.quiet): - self._suggest() - if self.options.mode != 'commit': - self._non_commit(result) - return False - else: - self._fail(result, can_force) - if self.options.pretend: - utilities.repoman_sez( - "\"So, you want to play it safe. Good call.\"\n") - return True - - - def perform(self, qa_output): - myautoadd = self._vcs_autoadd() - - self._vcs_deleted() - - changes = self.get_vcs_changed() - - mynew, mychanged, myremoved, no_expansion, expansion = changes - - # Manifests need to be regenerated after all other commits, so don't commit - # them now even if they have changed. - mymanifests = set() - myupdates = set() - for f in mychanged + mynew: - if "Manifest" == os.path.basename(f): - mymanifests.add(f) - else: - myupdates.add(f) - myupdates.difference_update(myremoved) - myupdates = list(myupdates) - mymanifests = list(mymanifests) - myheaders = [] - - commitmessage = self.options.commitmsg - if self.options.commitmsgfile: - try: - f = io.open( - _unicode_encode( - self.options.commitmsgfile, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], errors='replace') - commitmessage = f.read() - f.close() - del f - except (IOError, OSError) as e: - if e.errno == errno.ENOENT: - portage.writemsg( - "!!! File Not Found:" - " --commitmsgfile='%s'\n" % self.options.commitmsgfile) - else: - raise - if commitmessage[:9].lower() in ("cat/pkg: ",): - commitmessage = self.msg_prefix() + commitmessage[9:] - - if commitmessage is not None and commitmessage.strip(): - res, expl = self.verify_commit_message(commitmessage) - if not res: - print(bad("RepoMan does not like your commit message:")) - print(expl) - if self.options.force: - print('(but proceeding due to --force)') - else: - sys.exit(1) - else: - commitmessage = None - msg_qa_output = qa_output - initial_message = None - while True: - commitmessage = self.get_new_commit_message( - msg_qa_output, commitmessage) - res, expl = self.verify_commit_message(commitmessage) - if res: - break - else: - full_expl = '''Issues with the commit message were found. Please fix it or remove -the whole commit message to abort. - -''' + expl - msg_qa_output = ( - [' %s\n' % x for x in full_expl.splitlines()] - + qa_output) - - commitmessage = commitmessage.rstrip() - - # Update copyright for new and changed files - year = time.strftime('%Y', time.gmtime()) - for fn in chain(mynew, mychanged): - if fn.endswith('.diff') or fn.endswith('.patch'): - continue - update_copyright(fn, year, pretend=self.options.pretend) - - myupdates, broken_changelog_manifests = self.changelogs( - myupdates, mymanifests, myremoved, mychanged, myautoadd, - mynew, commitmessage) - - lines = commitmessage.splitlines() - lastline = lines[-1] - if len(lines) == 1 or re.match(r'^\S+:\s', lastline) is None: - commitmessage += '\n' - - commit_footer = self.get_commit_footer() - commitmessage += commit_footer - - print("* %s files being committed..." % green(str(len(myupdates))), end=' ') - - if not self.vcs_settings.needs_keyword_expansion: - # With some VCS types there's never any keyword expansion, so - # there's no need to regenerate manifests and all files will be - # committed in one big commit at the end. - logging.debug("VCS type doesn't need keyword expansion") - print() - elif not self.repo_settings.repo_config.thin_manifest: - logging.debug("perform: Calling thick_manifest()") - self.vcs_settings.changes.thick_manifest(myupdates, myheaders, - no_expansion, expansion) - - logging.info("myupdates: %s", myupdates) - logging.info("myheaders: %s", myheaders) - - uq = UserQuery(self.options) - if self.options.ask and uq.query('Commit changes?', True) != 'Yes': - print("* aborting commit.") - sys.exit(128 + signal.SIGINT) - - # Handle the case where committed files have keywords which - # will change and need a priming commit before the Manifest - # can be committed. - if (myupdates or myremoved) and myheaders: - self.priming_commit(myupdates, myremoved, commitmessage) - - # When files are removed and re-added, the cvs server will put /Attic/ - # inside the $Header path. This code detects the problem and corrects it - # so that the Manifest will generate correctly. See bug #169500. - # Use binary mode in order to avoid potential character encoding issues. - self.vcs_settings.changes.clear_attic(myheaders) - - if self.scanner.repolevel == 1: - utilities.repoman_sez( - "\"You're rather crazy... " - "doing the entire repository.\"\n") - - self.vcs_settings.changes.digest_regen(myupdates, myremoved, mymanifests, - self.scanner, broken_changelog_manifests) - - if self.repo_settings.sign_manifests: - self.sign_manifest(myupdates, myremoved, mymanifests) - - self.vcs_settings.changes.update_index(mymanifests, myupdates) - - self.add_manifest(mymanifests, myheaders, myupdates, myremoved, commitmessage) - - if self.options.quiet: - return - print() - if self.vcs_settings.vcs: - print("Commit complete.") - else: - print( - "repoman was too scared" - " by not seeing any familiar version control file" - " that he forgot to commit anything") - utilities.repoman_sez( - "\"If everyone were like you, I'd be out of business!\"\n") - return - - - def _suggest(self): - print() - if self.suggest['ignore_masked']: - print(bold( - "Note: use --without-mask to check " - "KEYWORDS on dependencies of masked packages")) - - if self.suggest['include_dev']: - print(bold( - "Note: use --include-dev (-d) to check " - "dependencies for 'dev' profiles")) - print() - - - def _non_commit(self, result): - if result['full']: - print(bold("Note: type \"repoman full\" for a complete listing.")) - if result['warn'] and not result['fail']: - if self.options.quiet: - print(bold("Non-Fatal QA errors found")) - else: - utilities.repoman_sez( - "\"You're only giving me a partial QA payment?\n" - " I'll take it this time, but I'm not happy.\"" - ) - elif not result['fail']: - if self.options.quiet: - print("No QA issues found") - else: - utilities.repoman_sez( - "\"If everyone were like you, I'd be out of business!\"") - elif result['fail']: - print(bad("Please fix these important QA issues first.")) - if not self.options.quiet: - utilities.repoman_sez( - "\"Make your QA payment on time" - " and you'll never see the likes of me.\"\n") - sys.exit(1) - - - def _fail(self, result, can_force): - if result['fail'] and can_force and self.options.force and not self.options.pretend: - utilities.repoman_sez( - " \"You want to commit even with these QA issues?\n" - " I'll take it this time, but I'm not happy.\"\n") - elif result['fail']: - if self.options.force and not can_force: - print(bad( - "The --force option has been disabled" - " due to extraordinary issues.")) - print(bad("Please fix these important QA issues first.")) - utilities.repoman_sez( - "\"Make your QA payment on time" - " and you'll never see the likes of me.\"\n") - sys.exit(1) - - - def _vcs_autoadd(self): - myunadded = self.vcs_settings.changes.unadded - myautoadd = [] - if myunadded: - for x in range(len(myunadded) - 1, -1, -1): - xs = myunadded[x].split("/") - if self.repo_settings.repo_config.find_invalid_path_char(myunadded[x]) != -1: - # The Manifest excludes this file, - # so it's safe to ignore. - del myunadded[x] - elif xs[-1] == "files": - print("!!! files dir is not added! Please correct this.") - sys.exit(-1) - elif xs[-1] == "Manifest": - # It's a manifest... auto add - myautoadd += [myunadded[x]] - del myunadded[x] - - if myunadded: - print(red( - "!!! The following files are in your local tree" - " but are not added to the master")) - print(red( - "!!! tree. Please remove them from the local tree" - " or add them to the master tree.")) - for x in myunadded: - print(" ", x) - print() - print() - sys.exit(1) - return myautoadd - - - def _vcs_deleted(self): - if self.vcs_settings.changes.has_deleted: - print(red( - "!!! The following files are removed manually" - " from your local tree but are not")) - print(red( - "!!! removed from the repository." - " Please remove them, using \"%s remove [FILES]\"." - % self.vcs_settings.vcs)) - for x in self.vcs_settings.changes.deleted: - print(" ", x) - print() - print() - sys.exit(1) - - - def get_vcs_changed(self): - '''Holding function which calls the approriate VCS module for the data''' - changes = self.vcs_settings.changes - # re-run the scan to pick up a newly modified Manifest file - logging.debug("RE-scanning for changes...") - changes.scan() - - if not changes.has_changes: - utilities.repoman_sez( - "\"Doing nothing is not always good for QA.\"") - print() - print("(Didn't find any changed files...)") - print() - sys.exit(1) - return (changes.new, changes.changed, changes.removed, - changes.no_expansion, changes.expansion) - - https_bugtrackers = frozenset([ - 'bitbucket.org', - 'bugs.gentoo.org', - 'github.com', - 'gitlab.com', - ]) - - def get_commit_footer(self): - portage_version = getattr(portage, "VERSION", None) - gpg_key = self.repoman_settings.get("PORTAGE_GPG_KEY", "") - dco_sob = self.repoman_settings.get("DCO_SIGNED_OFF_BY", "") - report_options = [] - if self.options.force: - report_options.append("--force") - if self.options.ignore_arches: - report_options.append("--ignore-arches") - if self.scanner.include_arches is not None: - report_options.append( - "--include-arches=\"%s\"" % - " ".join(sorted(self.scanner.include_arches))) - - if portage_version is None: - sys.stderr.write("Failed to insert portage version in message!\n") - sys.stderr.flush() - portage_version = "Unknown" - - # Common part of commit footer - commit_footer = "\n" - for tag, bug in chain( - (('Bug', x) for x in self.options.bug), - (('Closes', x) for x in self.options.closes)): - # case 1: pure number NNNNNN - if bug.isdigit(): - bug = 'https://bugs.gentoo.org/%s' % (bug, ) - else: - purl = urlsplit(bug) - qs = parse_qs(purl.query) - # case 2: long Gentoo bugzilla URL to shorten - if (purl.netloc == 'bugs.gentoo.org' and - purl.path == '/show_bug.cgi' and - tuple(qs.keys()) == ('id',)): - bug = urlunsplit(('https', purl.netloc, - qs['id'][-1], '', purl.fragment)) - # case 3: bug tracker w/ http -> https - elif (purl.scheme == 'http' and - purl.netloc in self.https_bugtrackers): - bug = urlunsplit(('https',) + purl[1:]) - commit_footer += "%s: %s\n" % (tag, bug) - - if dco_sob: - commit_footer += "Signed-off-by: %s\n" % (dco_sob, ) - - # Use new footer only for git (see bug #438364). - if self.vcs_settings.vcs in ["git"]: - commit_footer += "Package-Manager: Portage-%s, Repoman-%s" % ( - portage.VERSION, VERSION) - if report_options: - commit_footer += "\nRepoMan-Options: " + " ".join(report_options) - if self.repo_settings.sign_manifests: - commit_footer += "\nManifest-Sign-Key: %s" % (gpg_key, ) - else: - unameout = platform.system() + " " - if platform.system() in ["Darwin", "SunOS"]: - unameout += platform.processor() - else: - unameout += platform.machine() - commit_footer += "(Portage version: %s/%s/%s" % \ - (portage_version, self.vcs_settings.vcs, unameout) - if report_options: - commit_footer += ", RepoMan options: " + " ".join(report_options) - if self.repo_settings.sign_manifests: - commit_footer += ", signed Manifest commit with key %s" % \ - (gpg_key, ) - else: - commit_footer += ", unsigned Manifest commit" - commit_footer += ")" - return commit_footer - - - def changelogs(self, myupdates, mymanifests, myremoved, mychanged, myautoadd, - mynew, changelog_msg): - broken_changelog_manifests = [] - if self.options.echangelog in ('y', 'force'): - logging.info("checking for unmodified ChangeLog files") - committer_name = utilities.get_committer_name(env=self.repoman_settings) - for x in sorted(vcs_files_to_cps( - chain(myupdates, mymanifests, myremoved), - self.repo_settings.repodir, - self.scanner.repolevel, self.scanner.reposplit, self.scanner.categories)): - catdir, pkgdir = x.split("/") - checkdir = self.repo_settings.repodir + "/" + x - checkdir_relative = "" - if self.scanner.repolevel < 3: - checkdir_relative = os.path.join(pkgdir, checkdir_relative) - if self.scanner.repolevel < 2: - checkdir_relative = os.path.join(catdir, checkdir_relative) - checkdir_relative = os.path.join(".", checkdir_relative) - - changelog_path = os.path.join(checkdir_relative, "ChangeLog") - changelog_modified = changelog_path in self.scanner.changed.changelogs - if changelog_modified and self.options.echangelog != 'force': - continue - - # get changes for this package - cdrlen = len(checkdir_relative) - check_relative = lambda e: e.startswith(checkdir_relative) - split_relative = lambda e: e[cdrlen:] - clnew = list(map(split_relative, filter(check_relative, mynew))) - clremoved = list(map(split_relative, filter(check_relative, myremoved))) - clchanged = list(map(split_relative, filter(check_relative, mychanged))) - - # Skip ChangeLog generation if only the Manifest was modified, - # as discussed in bug #398009. - nontrivial_cl_files = set() - nontrivial_cl_files.update(clnew, clremoved, clchanged) - nontrivial_cl_files.difference_update(['Manifest']) - if not nontrivial_cl_files and self.options.echangelog != 'force': - continue - - new_changelog = utilities.UpdateChangeLog( - checkdir_relative, committer_name, changelog_msg, - os.path.join(self.repo_settings.repodir, 'skel.ChangeLog'), - catdir, pkgdir, - new=clnew, removed=clremoved, changed=clchanged, - pretend=self.options.pretend) - if new_changelog is None: - writemsg_level( - "!!! Updating the ChangeLog failed\n", - level=logging.ERROR, noiselevel=-1) - sys.exit(1) - - # if the ChangeLog was just created, add it to vcs - if new_changelog: - myautoadd.append(changelog_path) - # myautoadd is appended to myupdates below - else: - myupdates.append(changelog_path) - - if self.options.ask and not self.options.pretend: - # regenerate Manifest for modified ChangeLog (bug #420735) - self.repoman_settings["O"] = checkdir - digestgen(mysettings=self.repoman_settings, myportdb=self.repo_settings.portdb) - else: - broken_changelog_manifests.append(x) - - if myautoadd: - print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...") - self.vcs_settings.changes.add_items(myautoadd) - myupdates += myautoadd - return myupdates, broken_changelog_manifests - - - def add_manifest(self, mymanifests, myheaders, myupdates, myremoved, - commitmessage): - myfiles = mymanifests[:] - # If there are no header (SVN/CVS keywords) changes in - # the files, this Manifest commit must include the - # other (yet uncommitted) files. - if not myheaders: - myfiles += myupdates - myfiles += myremoved - myfiles.sort() - - commitmessagedir = tempfile.mkdtemp(".repoman.msg") - commitmessagefile = os.path.join(commitmessagedir, "COMMIT_EDITMSG") - with open(commitmessagefile, "wb") as mymsg: - mymsg.write(_unicode_encode(commitmessage)) - - retval = self.vcs_settings.changes.commit(myfiles, commitmessagefile) - # cleanup the commit message before possibly exiting - try: - shutil.rmtree(commitmessagedir) - except OSError: - pass - if retval != os.EX_OK: - writemsg_level( - "!!! Exiting on %s (shell) " - "error code: %s\n" % (self.vcs_settings.vcs, retval), - level=logging.ERROR, noiselevel=-1) - sys.exit(retval) - - - def priming_commit(self, myupdates, myremoved, commitmessage): - myfiles = myupdates + myremoved - commitmessagedir = tempfile.mkdtemp(".repoman.msg") - commitmessagefile = os.path.join(commitmessagedir, "COMMIT_EDITMSG") - with open(commitmessagefile, "wb") as mymsg: - mymsg.write(_unicode_encode(commitmessage)) - - separator = '-' * 78 - - print() - print(green("Using commit message:")) - print(green(separator)) - print(commitmessage) - print(green(separator)) - print() - - # Having a leading ./ prefix on file paths can trigger a bug in - # the cvs server when committing files to multiple directories, - # so strip the prefix. - myfiles = [f.lstrip("./") for f in myfiles] - - retval = self.vcs_settings.changes.commit(myfiles, commitmessagefile) - # cleanup the commit message before possibly exiting - try: - shutil.rmtree(commitmessagedir) - except OSError: - pass - if retval != os.EX_OK: - writemsg_level( - "!!! Exiting on %s (shell) " - "error code: %s\n" % (self.vcs_settings.vcs, retval), - level=logging.ERROR, noiselevel=-1) - sys.exit(retval) - - - def sign_manifest(self, myupdates, myremoved, mymanifests): - try: - for x in sorted(vcs_files_to_cps( - chain(myupdates, myremoved, mymanifests), - self.scanner.repo_settings.repodir, - self.scanner.repolevel, self.scanner.reposplit, self.scanner.categories)): - self.repoman_settings["O"] = os.path.join(self.repo_settings.repodir, x) - manifest_path = os.path.join(self.repoman_settings["O"], "Manifest") - if not need_signature(manifest_path): - continue - gpgsign(manifest_path, self.repoman_settings, self.options) - except portage.exception.PortageException as e: - portage.writemsg("!!! %s\n" % str(e)) - portage.writemsg("!!! Disabled FEATURES='sign'\n") - self.repo_settings.sign_manifests = False - - def msg_prefix(self): - prefix = "" - if self.scanner.repolevel > 1: - prefix = "/".join(self.scanner.reposplit[1:]) + ": " - return prefix - - def get_new_commit_message(self, qa_output, old_message=None): - msg_prefix = old_message or self.msg_prefix() - try: - editor = os.environ.get("EDITOR") - if editor and utilities.editor_is_executable(editor): - commitmessage = utilities.get_commit_message_with_editor( - editor, message=qa_output, prefix=msg_prefix) - else: - print("EDITOR is unset or invalid. Please set EDITOR to your preferred editor.") - print(bad("* no EDITOR found for commit message, aborting commit.")) - sys.exit(1) - except KeyboardInterrupt: - logging.fatal("Interrupted; exiting...") - sys.exit(1) - if (not commitmessage or not commitmessage.strip() - or commitmessage.strip() == msg_prefix): - print("* no commit message? aborting commit.") - sys.exit(1) - return commitmessage - - @staticmethod - def verify_commit_message(msg): - """ - Check whether the message roughly complies with GLEP66. Returns - (True, None) if it does, (False, <explanation>) if it does not. - """ - - problems = [] - paras = msg.strip().split('\n\n') - summary = paras.pop(0) - - if ':' not in summary: - problems.append('summary line must start with a logical unit name, e.g. "cat/pkg:"') - if '\n' in summary.strip(): - problems.append('commit message must start with a *single* line of summary, followed by empty line') - # accept 69 overall or unit+50, in case of very long package names - elif len(summary.strip()) > 69 and len(summary.split(':', 1)[-1]) > 50: - problems.append('summary line is too long (max 69 characters)') - - multiple_footers = False - gentoo_bug_used = False - bug_closes_without_url = False - body_too_long = False - - footer_re = re.compile(r'^[\w-]+:') - - found_footer = False - for p in paras: - lines = p.splitlines() - # if all lines look like footer material, we assume it's footer - # else, we assume it's body text - if all(footer_re.match(l) for l in lines if l.strip()): - # multiple footer-like blocks? - if found_footer: - multiple_footers = True - found_footer = True - for l in lines: - if l.startswith('Gentoo-Bug'): - gentoo_bug_used = True - elif l.startswith('Bug:') or l.startswith('Closes:'): - if 'http://' not in l and 'https://' not in l: - bug_closes_without_url = True - else: - for l in lines: - # we recommend wrapping at 72 but accept up to 80; - # do not complain if we have single word (usually - # it means very long URL) - if len(l.strip()) > 80 and len(l.split()) > 1: - body_too_long = True - - if multiple_footers: - problems.append('multiple footer-style blocks found, please combine them into one') - if gentoo_bug_used: - problems.append('please replace Gentoo-Bug with GLEP 66-compliant Bug/Closes') - if bug_closes_without_url: - problems.append('Bug/Closes footers require full URL') - if body_too_long: - problems.append('body lines should be wrapped at 72 (max 80) characters') - - if problems: - return (False, '\n'.join('- %s' % x for x in problems)) - return (True, None) diff --git a/repoman/pym/repoman/argparser.py b/repoman/pym/repoman/argparser.py deleted file mode 100644 index b87df95cd..000000000 --- a/repoman/pym/repoman/argparser.py +++ /dev/null @@ -1,243 +0,0 @@ -# repoman: Argument parser -# Copyright 2007-2017 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -"""This module contains functions used in Repoman to parse CLI arguments.""" - -import argparse -import logging -import sys - -# import our initialized portage instance -from repoman._portage import portage - -from portage import _unicode_decode -from portage import util - - -def parse_args(argv, repoman_default_opts): - """Use a customized optionParser to parse command line arguments for repoman - Args: - argv - a sequence of command line arguments - Returns: - (opts, args), just like a call to parser.parse_args() - """ - - argv = portage._decode_argv(argv) - - modes = { - 'commit': 'Run a scan then commit changes', - 'ci': 'Run a scan then commit changes', - 'fix': 'Fix simple QA issues (stray digests, missing digests)', - 'full': 'Scan directory tree and print all issues (not a summary)', - 'help': 'Show this screen', - 'manifest': 'Generate a Manifest (fetches files if necessary)', - 'manifest-check': 'Check Manifests for missing or incorrect digests', - 'scan': 'Scan directory tree for QA issues' - } - - output_choices = { - 'default': 'The normal output format', - 'column': 'Columnar output suitable for use with grep' - } - - mode_keys = list(modes) - mode_keys.sort() - - output_keys = sorted(output_choices) - - parser = argparse.ArgumentParser( - usage="repoman [options] [mode]", - description="Modes: %s" % " | ".join(mode_keys), - epilog="For more help consult the man page.") - - parser.add_argument( - '-a', '--ask', dest='ask', action='store_true', - default=False, - help='Request a confirmation before commiting') - - parser.add_argument( - '-b', '--bug', dest='bug', action='append', metavar='<BUG-NO|BUG-URL>', - default=[], - help=( - 'Mention a Gentoo or upstream bug in the commit footer; ' - 'takes either Gentoo bug number or full bug URL')) - - parser.add_argument( - '-c', '--closes', dest='closes', action='append', metavar='<PR-NO|PR-URL>', - default=[], - help=( - 'Adds a Closes footer to close GitHub pull request (or compatible); ' - 'takes either GitHub PR number or full PR URL')) - - parser.add_argument( - '-m', '--commitmsg', dest='commitmsg', - help='specify a commit message on the command line') - - parser.add_argument( - '-M', '--commitmsgfile', dest='commitmsgfile', - help='specify a path to a file that contains a commit message') - - parser.add_argument( - '--digest', choices=('y', 'n'), metavar='<y|n>', - help='Automatically update Manifest digests for modified files') - - parser.add_argument( - '-p', '--pretend', dest='pretend', default=False, - action='store_true', - help='don\'t commit or fix anything; just show what would be done') - - parser.add_argument( - '-q', '--quiet', dest="quiet", action="count", - default=0, - help='do not print unnecessary messages') - - parser.add_argument( - '--echangelog', choices=('y', 'n', 'force'), metavar="<y|n|force>", - help=( - 'for commit mode, call echangelog if ChangeLog is unmodified (or ' - 'regardless of modification if \'force\' is specified)')) - - parser.add_argument( - '--experimental-inherit', choices=('y', 'n'), metavar="<y|n>", - default='n', - help=( - 'Enable experimental inherit.missing checks which may misbehave' - ' when the internal eclass database becomes outdated')) - - parser.add_argument( - '--experimental-repository-modules', choices=('y', 'n'), metavar="<y|n>", - default='n', - help='Enable experimental repository modules') - - parser.add_argument( - '-f', '--force', dest='force', action='store_true', - default=False, - help='Commit with QA violations') - - parser.add_argument( - '-S', '--straight-to-stable', dest='straight_to_stable', - default=False, action='store_true', - help='Allow committing straight to stable') - - parser.add_argument( - '--vcs', dest='vcs', - help='Force using specific VCS instead of autodetection') - - parser.add_argument( - '-v', '--verbose', dest="verbosity", action='count', - help='be very verbose in output', default=0) - - parser.add_argument( - '-V', '--version', dest='version', action='store_true', - help='show version info') - - parser.add_argument( - '-x', '--xmlparse', dest='xml_parse', action='store_true', - default=False, - help='forces the metadata.xml parse check to be carried out') - - parser.add_argument( - '--if-modified', choices=('y', 'n'), default='n', - metavar="<y|n>", - help='only check packages that have uncommitted modifications') - - parser.add_argument( - '-i', '--ignore-arches', dest='ignore_arches', action='store_true', - default=False, - help='ignore arch-specific failures (where arch != host)') - - parser.add_argument( - "--ignore-default-opts", - action="store_true", - help="do not use the REPOMAN_DEFAULT_OPTS environment variable") - - parser.add_argument( - '-I', '--ignore-masked', dest='ignore_masked', action='store_true', - default=False, - help='ignore masked packages (not allowed with commit mode)') - - parser.add_argument( - '--include-arches', - dest='include_arches', metavar='ARCHES', action='append', - help=( - 'A space separated list of arches used to ' - 'filter the selection of profiles for dependency checks')) - - parser.add_argument( - '-d', '--include-dev', dest='include_dev', action='store_true', - default=False, - help='include dev profiles in dependency checks') - - parser.add_argument( - '-e', '--include-exp-profiles', choices=('y', 'n'), metavar='<y|n>', - default=False, - help='include exp profiles in dependency checks') - - parser.add_argument( - '--unmatched-removal', dest='unmatched_removal', action='store_true', - default=False, - help=( - 'enable strict checking of package.mask and package.unmask files' - ' for unmatched removal atoms')) - - parser.add_argument( - '--without-mask', dest='without_mask', action='store_true', - default=False, - help=( - 'behave as if no package.mask entries exist' - ' (not allowed with commit mode)')) - - parser.add_argument( - '--output-style', dest='output_style', choices=output_keys, - help='select output type', default='default') - - parser.add_argument( - '--mode', dest='mode', choices=mode_keys, - help='specify which mode repoman will run in (default=full)') - - opts, args = parser.parse_known_args(argv[1:]) - - if not opts.ignore_default_opts: - default_opts = util.shlex_split(repoman_default_opts) - if default_opts: - opts, args = parser.parse_known_args(default_opts + sys.argv[1:]) - - if opts.mode == 'help': - parser.print_help(short=False) - - for arg in args: - if arg in modes: - if not opts.mode: - opts.mode = arg - break - else: - parser.error("invalid mode: %s" % arg) - - if not opts.mode: - opts.mode = 'full' - - if opts.mode == 'ci': - opts.mode = 'commit' # backwards compat shortcut - - # Use verbosity and quiet options to appropriately fiddle with the loglevel - for val in range(opts.verbosity): - logger = logging.getLogger() - logger.setLevel(logger.getEffectiveLevel() - 10) - - for val in range(opts.quiet): - logger = logging.getLogger() - logger.setLevel(logger.getEffectiveLevel() + 10) - - if opts.mode == 'commit' and opts.commitmsg: - opts.commitmsg = _unicode_decode(opts.commitmsg) - - if opts.mode == 'commit' and not (opts.force or opts.pretend): - if opts.ignore_masked: - opts.ignore_masked = False - logging.warn('Commit mode automatically disables --ignore-masked') - if opts.without_mask: - opts.without_mask = False - logging.warn('Commit mode automatically disables --without-mask') - - return (opts, args) diff --git a/repoman/pym/repoman/check_missingslot.py b/repoman/pym/repoman/check_missingslot.py deleted file mode 100644 index 4a3c57b2c..000000000 --- a/repoman/pym/repoman/check_missingslot.py +++ /dev/null @@ -1,30 +0,0 @@ -# -*- coding:utf-8 -*- -# repoman: missing slot check -# Copyright 2014 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -"""This module contains the check used to find missing slot values -in dependencies.""" - -from portage.eapi import eapi_has_slot_operator - -def check_missingslot(atom, mytype, eapi, portdb, qatracker, relative_path, my_aux): - # If no slot or slot operator is specified in RDEP... - if (not atom.blocker and not atom.slot and not atom.slot_operator - and mytype == 'RDEPEND' and eapi_has_slot_operator(eapi)): - # Check whether it doesn't match more than one. - atom_matches = portdb.xmatch("match-all", atom) - dep_slots = frozenset( - portdb.aux_get(cpv, ['SLOT'])[0].split('/')[0] - for cpv in atom_matches) - - if len(dep_slots) > 1: - # See if it is a DEPEND as well. It's a very simple & dumb - # check but should suffice for catching it. - depend = my_aux['DEPEND'].split() - if atom not in depend: - return - - qatracker.add_error("dependency.missingslot", relative_path + - ": %s: '%s' matches more than one slot, please specify an explicit slot and/or use the := or :* slot operator" % - (mytype, atom)) diff --git a/repoman/pym/repoman/checks/__init__.py b/repoman/pym/repoman/checks/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/repoman/pym/repoman/checks/__init__.py +++ /dev/null diff --git a/repoman/pym/repoman/checks/herds/__init__.py b/repoman/pym/repoman/checks/herds/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/repoman/pym/repoman/checks/herds/__init__.py +++ /dev/null diff --git a/repoman/pym/repoman/checks/herds/herdbase.py b/repoman/pym/repoman/checks/herds/herdbase.py deleted file mode 100644 index ebe6a19b4..000000000 --- a/repoman/pym/repoman/checks/herds/herdbase.py +++ /dev/null @@ -1,135 +0,0 @@ -# -*- coding: utf-8 -*- -# repoman: Herd database analysis -# Copyright 2010-2013 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 or later - -from __future__ import print_function, unicode_literals - -import errno -import xml.etree.ElementTree -try: - from xml.parsers.expat import ExpatError -except (SystemExit, KeyboardInterrupt): - raise -except (ImportError, SystemError, RuntimeError, Exception): - # broken or missing xml support - # https://bugs.python.org/issue14988 - # This means that python is built without xml support. - # We tolerate global scope import failures for optional - # modules, so that ImportModulesTestCase can succeed (or - # possibly alert us about unexpected import failures). - pass - -from portage import _encodings, _unicode_encode -from portage.exception import FileNotFound, ParseError, PermissionDenied -from portage import os - -from repoman.errors import err - -__all__ = [ - "make_herd_base", "get_herd_base" -] - - -def _make_email(nick_name): - if not nick_name.endswith('@gentoo.org'): - nick_name = nick_name + '@gentoo.org' - return nick_name - - -class HerdBase(object): - def __init__(self, herd_to_emails, all_emails): - self.herd_to_emails = herd_to_emails - self.all_emails = all_emails - - def known_herd(self, herd_name): - return herd_name in self.herd_to_emails - - def known_maintainer(self, nick_name): - return _make_email(nick_name) in self.all_emails - - def maintainer_in_herd(self, nick_name, herd_name): - return _make_email(nick_name) in self.herd_to_emails[herd_name] - - -class _HerdsTreeBuilder(xml.etree.ElementTree.TreeBuilder): - """ - Implements doctype() as required to avoid deprecation warnings with - >=python-2.7. - """ - def doctype(self, name, pubid, system): - pass - - -def make_herd_base(filename): - herd_to_emails = dict() - all_emails = set() - - try: - xml_tree = xml.etree.ElementTree.parse( - _unicode_encode( - filename, encoding=_encodings['fs'], errors='strict'), - parser=xml.etree.ElementTree.XMLParser( - target=_HerdsTreeBuilder())) - except ExpatError as e: - raise ParseError("metadata.xml: %s" % (e,)) - except EnvironmentError as e: - func_call = "open('%s')" % filename - if e.errno == errno.EACCES: - raise PermissionDenied(func_call) - elif e.errno == errno.ENOENT: - raise FileNotFound(filename) - raise - - herds = xml_tree.findall('herd') - for h in herds: - _herd_name = h.find('name') - if _herd_name is None: - continue - herd_name = _herd_name.text.strip() - del _herd_name - - maintainers = h.findall('maintainer') - herd_emails = set() - for m in maintainers: - _m_email = m.find('email') - if _m_email is None: - continue - m_email = _m_email.text.strip() - - herd_emails.add(m_email) - all_emails.add(m_email) - - herd_to_emails[herd_name] = herd_emails - - return HerdBase(herd_to_emails, all_emails) - - -def get_herd_base(repoman_settings): - try: - herd_base = make_herd_base( - os.path.join(repoman_settings["PORTDIR"], "metadata/herds.xml")) - except (EnvironmentError, ParseError, PermissionDenied) as e: - err(str(e)) - except FileNotFound: - # TODO: Download as we do for metadata.dtd, but add a way to - # disable for non-gentoo repoman users who may not have herds. - herd_base = None - return herd_base - - -if __name__ == '__main__': - h = make_herd_base('/usr/portage/metadata/herds.xml') - - assert(h.known_herd('sound')) - assert(not h.known_herd('media-sound')) - - assert(h.known_maintainer('sping')) - assert(h.known_maintainer('sping@gentoo.org')) - assert(not h.known_maintainer('portage')) - - assert(h.maintainer_in_herd('zmedico@gentoo.org', 'tools-portage')) - assert(not h.maintainer_in_herd('pva@gentoo.org', 'tools-portage')) - - import pprint - pprint.pprint(h.herd_to_emails) diff --git a/repoman/pym/repoman/checks/herds/metadata.py b/repoman/pym/repoman/checks/herds/metadata.py deleted file mode 100644 index b4a433ed7..000000000 --- a/repoman/pym/repoman/checks/herds/metadata.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- coding:utf-8 -*- - - -class UnknownHerdsError(ValueError): - def __init__(self, herd_names): - _plural = len(herd_names) != 1 - super(UnknownHerdsError, self).__init__( - 'Unknown %s %s' % ( - _plural and 'herds' or 'herd', - ','.join('"%s"' % e for e in herd_names))) - - -def check_metadata_herds(xml_tree, herd_base): - herd_nodes = xml_tree.findall('herd') - unknown_herds = [ - name for name in ( - e.text.strip() for e in herd_nodes if e.text is not None) - if not herd_base.known_herd(name)] - - if unknown_herds: - raise UnknownHerdsError(unknown_herds) - - -def check_metadata(xml_tree, herd_base): - if herd_base is not None: - check_metadata_herds(xml_tree, herd_base) diff --git a/repoman/pym/repoman/config.py b/repoman/pym/repoman/config.py deleted file mode 100644 index 578bbccde..000000000 --- a/repoman/pym/repoman/config.py +++ /dev/null @@ -1,159 +0,0 @@ -# -*- coding:utf-8 -*- - -import copy -import itertools -import json -import os -import stat - -import yaml - -try: - FileNotFoundError -except NameError: - FileNotFoundError = EnvironmentError - - -class ConfigError(Exception): - """Raised when a config file fails to load""" - pass - - -def merge_config(base, head): - """ - Merge two JSON or YAML documents into a single object. Arrays are - merged by extension. If dissimilar types are encountered, then the - head value overwrites the base value. - """ - - if isinstance(head, dict): - if not isinstance(base, dict): - return copy.deepcopy(head) - - result = {} - for k in itertools.chain(head, base): - try: - result[k] = merge_config(base[k], head[k]) - except KeyError: - try: - result[k] = copy.deepcopy(head[k]) - except KeyError: - result[k] = copy.deepcopy(base[k]) - - elif isinstance(head, list): - result = [] - if not isinstance(base, list): - result.extend(copy.deepcopy(x) for x in head) - else: - if any(isinstance(x, (dict, list)) for x in itertools.chain(head, base)): - # merge items with identical indexes - for x, y in zip(base, head): - if isinstance(x, (dict, list)): - result.append(merge_config(x, y)) - else: - # head overwrites base (preserving index) - result.append(copy.deepcopy(y)) - # copy remaining items from the longer list - if len(base) != len(head): - if len(base) > len(head): - result.extend(copy.deepcopy(x) for x in base[len(head):]) - else: - result.extend(copy.deepcopy(x) for x in head[len(base):]) - else: - result.extend(copy.deepcopy(x) for x in base) - result.extend(copy.deepcopy(x) for x in head) - - else: - result = copy.deepcopy(head) - - return result - -def _yaml_load(filename): - """ - Load filename as YAML and return a dict. Raise ConfigError if - it fails to load. - """ - with open(filename, 'rt') as f: - try: - return yaml.safe_load(f) - except yaml.parser.ParserError as e: - raise ConfigError("{}: {}".format(filename, e)) - -def _json_load(filename): - """ - Load filename as JSON and return a dict. Raise ConfigError if - it fails to load. - """ - with open(filename, 'rt') as f: - try: - return json.load(f) #nosec - except ValueError as e: - raise ConfigError("{}: {}".format(filename, e)) - -def iter_files(files_dirs): - """ - Iterate over nested file paths in lexical order. - """ - stack = list(reversed(files_dirs)) - while stack: - location = stack.pop() - try: - st = os.stat(location) - except FileNotFoundError: - continue - - if stat.S_ISDIR(st.st_mode): - stack.extend(os.path.join(location, x) - for x in sorted(os.listdir(location), reverse=True)) - - elif stat.S_ISREG(st.st_mode): - yield location - -def load_config(conf_dirs, file_extensions=None, valid_versions=None): - """ - Load JSON and/or YAML files from a directories, and merge them together - into a single object. - - @param conf_dirs: ordered iterable of directories to load the config from - @param file_extensions: Optional list of file extension types to load - @param valid_versions: list of compatible file versions allowed - @returns: the stacked config - """ - - result = {} - for filename in iter_files(conf_dirs): - if file_extensions is not None and not filename.endswith(file_extensions): - continue - - loaders = [] - extension = filename.rsplit('.', 1)[1] - if extension in ['json']: - loaders.append(_json_load) - elif extension in ['yml', 'yaml']: - loaders.append(_yaml_load) - - config = None - exception = None - for loader in loaders: - try: - config = loader(filename) or {} - except ConfigError as e: - exception = e - else: - break - - if config is None: - print("Repoman.config.load_config(), Error loading file: %s" % filename) - print(" Aborting...") - raise exception - - if config: - if config['version'] not in valid_versions: - raise ConfigError( - "Invalid file version: %s in: %s\nPlease upgrade to " - ">=app-portage/repoman-%s, current valid API versions: %s" - % (config['version'], filename, - config['repoman_version'], valid_versions)) - result = merge_config(result, config) - - return result diff --git a/repoman/pym/repoman/copyrights.py b/repoman/pym/repoman/copyrights.py deleted file mode 100644 index 94257c942..000000000 --- a/repoman/pym/repoman/copyrights.py +++ /dev/null @@ -1,122 +0,0 @@ -# -*- coding:utf-8 -*- - - -import difflib -import io -import re -from tempfile import mkstemp - -from portage import _encodings -from portage import _unicode_decode -from portage import _unicode_encode -from portage import os -from portage import shutil -from portage import util - - -_copyright_re1 = \ - re.compile(br'^(# Copyright \d\d\d\d)-\d\d\d\d( Gentoo Foundation)\b') -_copyright_re2 = \ - re.compile(br'^(# Copyright )(\d\d\d\d)( Gentoo Foundation)\b') - - -class _copyright_repl(object): - __slots__ = ('year',) - - def __init__(self, year): - self.year = year - - def __call__(self, matchobj): - if matchobj.group(2) == self.year: - return matchobj.group(0) - else: - return matchobj.group(1) + matchobj.group(2) + \ - b'-' + self.year + matchobj.group(3) - - -def update_copyright_year(year, line): - """ - These two regexes are taken from echangelog - update_copyright(), except that we don't hardcode - 1999 here (in order to be more generic). - """ - is_bytes = isinstance(line, bytes) - if is_bytes: - if not line.startswith(b'# Copyright '): - return line - else: - if not line.startswith('# Copyright '): - return line - - year = _unicode_encode(year) - line = _unicode_encode(line) - - line = _copyright_re1.sub(br'\1-' + year + br'\2', line) - line = _copyright_re2.sub(_copyright_repl(year), line) - if not is_bytes: - line = _unicode_decode(line) - return line - - -def update_copyright(fn_path, year, pretend=False): - """ - Check file for a Copyright statement, and update its year. The - patterns used for replacing copyrights are taken from echangelog. - Only the first lines of each file that start with a hash ('#') are - considered, until a line is found that doesn't start with a hash. - Files are read and written in binary mode, so that this function - will work correctly with files encoded in any character set, as - long as the copyright statements consist of plain ASCII. - """ - - try: - fn_hdl = io.open(_unicode_encode( - fn_path, encoding=_encodings['fs'], errors='strict'), - mode='rb') - except EnvironmentError: - return - - orig_header = [] - new_header = [] - - for line in fn_hdl: - line_strip = line.strip() - orig_header.append(line) - if not line_strip or line_strip[:1] != b'#': - new_header.append(line) - break - - line = update_copyright_year(year, line) - new_header.append(line) - - difflines = 0 - for diffline in difflib.unified_diff( - [_unicode_decode(diffline) for diffline in orig_header], - [_unicode_decode(diffline) for diffline in new_header], - fromfile=fn_path, tofile=fn_path, n=0): - util.writemsg_stdout(diffline, noiselevel=-1) - difflines += 1 - util.writemsg_stdout("\n", noiselevel=-1) - - # unified diff has three lines to start with - if difflines > 3 and not pretend: - # write new file with changed header - f, fnnew_path = mkstemp() - f = io.open(f, mode='wb') - for line in new_header: - f.write(line) - for line in fn_hdl: - f.write(line) - f.close() - try: - fn_stat = os.stat(fn_path) - except OSError: - fn_stat = None - - shutil.move(fnnew_path, fn_path) - - if fn_stat is None: - util.apply_permissions(fn_path, mode=0o644) - else: - util.apply_stat_permissions(fn_path, fn_stat) - fn_hdl.close() diff --git a/repoman/pym/repoman/errors.py b/repoman/pym/repoman/errors.py deleted file mode 100644 index 9cf113ba0..000000000 --- a/repoman/pym/repoman/errors.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding:utf-8 -*- - -from __future__ import print_function, unicode_literals - -import sys - - -def warn(txt): - print("repoman: " + txt) - - -def err(txt): - warn(txt) - sys.exit(1) - - -def caterror(catdir, repodir): - err( - "%s is not an official category." - " Skipping QA checks in this directory.\n" - "Please ensure that you add %s to %s/profiles/categories\n" - "if it is a new category." % (catdir, catdir, repodir)) diff --git a/repoman/pym/repoman/gpg.py b/repoman/pym/repoman/gpg.py deleted file mode 100644 index a3c12b3c9..000000000 --- a/repoman/pym/repoman/gpg.py +++ /dev/null @@ -1,82 +0,0 @@ -# -*- coding:utf-8 -*- - -from __future__ import print_function, unicode_literals - -import errno -import logging -import subprocess -import sys - -import portage -from portage import os -from portage import _encodings -from portage import _unicode_encode -from portage.exception import MissingParameter -from portage.process import find_binary - - -# Setup the GPG commands -def gpgsign(filename, repoman_settings, options): - gpgcmd = repoman_settings.get("PORTAGE_GPG_SIGNING_COMMAND") - if gpgcmd in [None, '']: - raise MissingParameter("PORTAGE_GPG_SIGNING_COMMAND is unset!" - " Is make.globals missing?") - if "${PORTAGE_GPG_KEY}" in gpgcmd and \ - "PORTAGE_GPG_KEY" not in repoman_settings: - raise MissingParameter("PORTAGE_GPG_KEY is unset!") - if "${PORTAGE_GPG_DIR}" in gpgcmd: - if "PORTAGE_GPG_DIR" not in repoman_settings: - repoman_settings["PORTAGE_GPG_DIR"] = \ - os.path.expanduser("~/.gnupg") - logging.info( - "Automatically setting PORTAGE_GPG_DIR to '%s'" % - repoman_settings["PORTAGE_GPG_DIR"]) - else: - repoman_settings["PORTAGE_GPG_DIR"] = \ - os.path.expanduser(repoman_settings["PORTAGE_GPG_DIR"]) - if not os.access(repoman_settings["PORTAGE_GPG_DIR"], os.X_OK): - raise portage.exception.InvalidLocation( - "Unable to access directory: PORTAGE_GPG_DIR='%s'" % - repoman_settings["PORTAGE_GPG_DIR"]) - gpgvars = {"FILE": filename} - for k in ("PORTAGE_GPG_DIR", "PORTAGE_GPG_KEY"): - v = repoman_settings.get(k) - if v is not None: - gpgvars[k] = v - gpgcmd = portage.util.varexpand(gpgcmd, mydict=gpgvars) - if options.pretend: - print("(" + gpgcmd + ")") - else: - # Encode unicode manually for bug #310789. - gpgcmd = portage.util.shlex_split(gpgcmd) - - if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \ - not os.path.isabs(gpgcmd[0]): - # Python 3.1 _execvp throws TypeError for non-absolute executable - # path passed as bytes (see https://bugs.python.org/issue8513). - fullname = find_binary(gpgcmd[0]) - if fullname is None: - raise portage.exception.CommandNotFound(gpgcmd[0]) - gpgcmd[0] = fullname - - gpgcmd = [ - _unicode_encode(arg, encoding=_encodings['fs'], errors='strict') - for arg in gpgcmd] - rValue = subprocess.call(gpgcmd) - if rValue == os.EX_OK: - os.rename(filename + ".asc", filename) - else: - raise portage.exception.PortageException( - "!!! gpg exited with '" + str(rValue) + "' status") - -def need_signature(filename): - try: - with open( - _unicode_encode( - filename, encoding=_encodings['fs'], errors='strict'), - 'rb') as f: - return b"BEGIN PGP SIGNED MESSAGE" not in f.readline() - except IOError as e: - if e.errno in (errno.ENOENT, errno.ESTALE): - return False - raise diff --git a/repoman/pym/repoman/main.py b/repoman/pym/repoman/main.py deleted file mode 100755 index 81e2ff61e..000000000 --- a/repoman/pym/repoman/main.py +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/env python -# -*- coding:utf-8 -*- -# Copyright 1999-2017 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from __future__ import print_function, unicode_literals - -import io -import logging -import sys - -# import our centrally initialized portage instance -from repoman._portage import portage -portage._internal_caller = True -portage._disable_legacy_globals() - - -from portage import os -import portage.checksum -import portage.const -import portage.repository.config -from portage.output import create_color_func, nocolor -from portage.output import ConsoleStyleFile, StyleWriter -from portage.util import formatter -from portage.util.futures.extendedfutures import ( - ExtendedFuture, -) - -from repoman.actions import Actions -from repoman.argparser import parse_args -from repoman.qa_data import QAData -from repoman.qa_data import format_qa_output, format_qa_output_column -from repoman.repos import RepoSettings -from repoman.scanner import Scanner -from repoman import utilities -from repoman.modules.vcs.settings import VCSSettings -from repoman import VERSION - -if sys.hexversion >= 0x3000000: - basestring = str - -bad = create_color_func("BAD") - -# A sane umask is needed for files that portage creates. -os.umask(0o22) - -LOGLEVEL = logging.WARNING -portage.util.initialize_logger(LOGLEVEL) - -VALID_VERSIONS = [1,] - -def repoman_main(argv): - config_root = os.environ.get("PORTAGE_CONFIGROOT") - repoman_settings = portage.config(config_root=config_root, local_config=False) - repoman_settings.valid_versions = VALID_VERSIONS - - if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \ - repoman_settings.get('TERM') == 'dumb' or \ - not sys.stdout.isatty(): - nocolor() - - options, arguments = parse_args( - sys.argv, repoman_settings.get("REPOMAN_DEFAULT_OPTS", "")) - - if options.version: - print("Repoman", VERSION, "(portage-%s)" % portage.VERSION) - sys.exit(0) - - logger = logging.getLogger() - - if options.verbosity > 0: - logger.setLevel(LOGLEVEL - 10 * options.verbosity) - else: - logger.setLevel(LOGLEVEL) - - # Set this to False when an extraordinary issue (generally - # something other than a QA issue) makes it impossible to - # commit (like if Manifest generation fails). - can_force = ExtendedFuture(True) - - portdir, portdir_overlay, mydir = utilities.FindPortdir(repoman_settings) - if portdir is None: - sys.exit(1) - - myreporoot = os.path.basename(portdir_overlay) - myreporoot += mydir[len(portdir_overlay):] - - # avoid a circular parameter repo_settings - vcs_settings = VCSSettings(options, repoman_settings) - qadata = QAData() - - logging.debug("repoman_main: RepoSettings init") - repo_settings = RepoSettings( - config_root, portdir, portdir_overlay, - repoman_settings, vcs_settings, options, qadata) - repoman_settings = repo_settings.repoman_settings - repoman_settings.valid_versions = VALID_VERSIONS - - # Now set repo_settings - vcs_settings.repo_settings = repo_settings - # set QATracker qacats, qawarnings - vcs_settings.qatracker.qacats = repo_settings.qadata.qacats - vcs_settings.qatracker.qawarnings = repo_settings.qadata.qawarnings - logging.debug("repoman_main: vcs_settings done") - logging.debug("repoman_main: qadata: %s", repo_settings.qadata) - - if 'digest' in repoman_settings.features and options.digest != 'n': - options.digest = 'y' - - logging.debug("vcs: %s" % (vcs_settings.vcs,)) - logging.debug("repo config: %s" % (repo_settings.repo_config,)) - logging.debug("options: %s" % (options,)) - - # It's confusing if these warnings are displayed without the user - # being told which profile they come from, so disable them. - env = os.environ.copy() - env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn' - - # Perform the main checks - scanner = Scanner(repo_settings, myreporoot, config_root, options, - vcs_settings, mydir, env) - scanner.scan_pkgs(can_force) - - if options.if_modified == "y" and len(scanner.effective_scanlist) < 1: - logging.warning("--if-modified is enabled, but no modified packages were found!") - - result = { - # fail will be true if we have failed in at least one non-warning category - 'fail': 0, - # warn will be true if we tripped any warnings - 'warn': 0, - # full will be true if we should print a "repoman full" informational message - 'full': options.mode != 'full', - } - - # early out for manifest generation - if options.mode == "manifest": - sys.exit(result['fail']) - - for x in qadata.qacats: - if x not in vcs_settings.qatracker.fails: - continue - result['warn'] = 1 - if x not in qadata.qawarnings: - result['fail'] = 1 - - if result['fail'] or \ - (result['warn'] and not (options.quiet or options.mode == "scan")): - result['full'] = 0 - - commitmessage = None - if options.commitmsg: - commitmessage = options.commitmsg - elif options.commitmsgfile: - # we don't need the actual text of the commit message here - # the filename will do for the next code block - commitmessage = options.commitmsgfile - - # Save QA output so that it can be conveniently displayed - # in $EDITOR while the user creates a commit message. - # Otherwise, the user would not be able to see this output - # once the editor has taken over the screen. - qa_output = io.StringIO() - style_file = ConsoleStyleFile(sys.stdout) - if options.mode == 'commit' and \ - (not commitmessage or not commitmessage.strip()): - style_file.write_listener = qa_output - console_writer = StyleWriter(file=style_file, maxcol=9999) - console_writer.style_listener = style_file.new_styles - - f = formatter.AbstractFormatter(console_writer) - - format_outputs = { - 'column': format_qa_output_column, - 'default': format_qa_output - } - - format_output = format_outputs.get( - options.output_style, format_outputs['default']) - format_output(f, vcs_settings.qatracker.fails, result['full'], - result['fail'], options, qadata.qawarnings) - - style_file.flush() - del console_writer, f, style_file - qa_output = qa_output.getvalue() - qa_output = qa_output.splitlines(True) - - # output the results - actions = Actions(repo_settings, options, scanner, vcs_settings) - if actions.inform(can_force.get(), result): - # perform any other actions - actions.perform(qa_output) - - sys.exit(0) diff --git a/repoman/pym/repoman/metadata.py b/repoman/pym/repoman/metadata.py deleted file mode 100644 index 11ec1aaf8..000000000 --- a/repoman/pym/repoman/metadata.py +++ /dev/null @@ -1,128 +0,0 @@ -# -*- coding:utf-8 -*- - -from __future__ import print_function, unicode_literals - -import errno -import logging -import sys -import tempfile -import time - -try: - from urllib.parse import urlparse -except ImportError: - from urlparse import urlparse - - -# import our initialized portage instance -from repoman._portage import portage - -from portage import os -from portage import shutil -from portage.output import green - -if sys.hexversion >= 0x3000000: - basestring = str - -if sys.hexversion >= 0x3000000: - basestring = str - -# Note: This URI is hardcoded in all metadata.xml files. We can't -# change it without updating all the xml files in the tree. -metadata_dtd_uri = 'https://www.gentoo.org/dtd/metadata.dtd' -metadata_xsd_uri = 'https://www.gentoo.org/xml-schema/metadata.xsd' -# force refetch if the local copy creation time is older than this -metadata_xsd_ctime_interval = 60 * 60 * 24 * 7 # 7 days - - -def fetch_metadata_xsd(metadata_xsd, repoman_settings): - """ - Fetch metadata.xsd if it doesn't exist or the ctime is older than - metadata_xsd_ctime_interval. - @rtype: bool - @return: True if successful, otherwise False - """ - - must_fetch = True - metadata_xsd_st = None - current_time = int(time.time()) - try: - metadata_xsd_st = os.stat(metadata_xsd) - except EnvironmentError as e: - if e.errno not in (errno.ENOENT, errno.ESTALE): - raise - del e - else: - # Trigger fetch if metadata.xsd mtime is old or clock is wrong. - if abs(current_time - metadata_xsd_st.st_ctime) \ - < metadata_xsd_ctime_interval: - must_fetch = False - - if must_fetch: - print() - print( - "%s the local copy of metadata.xsd " - "needs to be refetched, doing that now" % green("***")) - print() - parsed_url = urlparse(metadata_xsd_uri) - setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper() - fcmd = repoman_settings.get(setting) - if not fcmd: - fcmd = repoman_settings.get('FETCHCOMMAND') - if not fcmd: - logging.error("FETCHCOMMAND is unset") - return False - - destdir = repoman_settings["DISTDIR"] - fd, metadata_xsd_tmp = tempfile.mkstemp( - prefix='metadata.xsd.', dir=destdir) - os.close(fd) - - try: - if not portage.getbinpkg.file_get( - metadata_xsd_uri, destdir, fcmd=fcmd, - filename=os.path.basename(metadata_xsd_tmp)): - logging.error( - "failed to fetch metadata.xsd from '%s'" % metadata_xsd_uri) - return False - - try: - portage.util.apply_secpass_permissions( - metadata_xsd_tmp, - gid=portage.data.portage_gid, mode=0o664, mask=0o2) - except portage.exception.PortageException: - pass - - shutil.move(metadata_xsd_tmp, metadata_xsd) - finally: - try: - os.unlink(metadata_xsd_tmp) - except OSError: - pass - - return True - - -def get_metadata_xsd(repo_settings): - '''Locate and or fetch the metadata.xsd file - - @param repo_settings: RepoSettings instance - @returns: path to the metadata.xsd file - ''' - metadata_xsd = None - paths = list(repo_settings.repo_config.eclass_db.porttrees) - paths.reverse() - # add the test copy - paths.append("/usr/lib/portage/cnf/") - for path in paths: - path = os.path.join(path, 'metadata/xml-schema/metadata.xsd') - if os.path.exists(path): - metadata_xsd = path - break - if metadata_xsd is None: - metadata_xsd = os.path.join( - repo_settings.repoman_settings["DISTDIR"], 'metadata.xsd' - ) - - fetch_metadata_xsd(metadata_xsd, repo_settings.repoman_settings) - return metadata_xsd diff --git a/repoman/pym/repoman/modules/__init__.py b/repoman/pym/repoman/modules/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/repoman/pym/repoman/modules/__init__.py +++ /dev/null diff --git a/repoman/pym/repoman/modules/commit/__init__.py b/repoman/pym/repoman/modules/commit/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/repoman/pym/repoman/modules/commit/__init__.py +++ /dev/null diff --git a/repoman/pym/repoman/modules/commit/manifest.py b/repoman/pym/repoman/modules/commit/manifest.py deleted file mode 100644 index b338a5b40..000000000 --- a/repoman/pym/repoman/modules/commit/manifest.py +++ /dev/null @@ -1,115 +0,0 @@ -# -*- coding:utf-8 -*- - -import logging -import sys - -# import our initialized portage instance -from repoman._portage import portage - -from portage import os -from portage.package.ebuild.digestgen import digestgen -from portage.util import writemsg_level - - -class Manifest(object): - '''Creates as well as checks pkg Manifest entries/files''' - - def __init__(self, **kwargs): - '''Class init - - @param options: the run time cli options - @param portdb: portdb instance - @param repo_settings: repository settings instance - ''' - self.options = kwargs.get('options') - self.portdb = kwargs.get('portdb') - self.repoman_settings = kwargs.get('repo_settings').repoman_settings - self.generated_manifest = False - - def update_manifest(self, checkdir): - '''Perform a manifest generation for the pkg - - @param checkdir: the current package directory - @returns: dictionary - ''' - self.generated_manifest = False - failed = False - self.auto_assumed = set() - fetchlist_dict = portage.FetchlistDict( - checkdir, self.repoman_settings, self.portdb) - if self.options.mode == 'manifest' and self.options.force: - self._discard_dist_digests(checkdir, fetchlist_dict) - self.repoman_settings["O"] = checkdir - try: - self.generated_manifest = digestgen( - mysettings=self.repoman_settings, myportdb=self.portdb) - except portage.exception.PermissionDenied as e: - self.generated_manifest = False - writemsg_level( - "!!! Permission denied: '%s'\n" % (e,), - level=logging.ERROR, noiselevel=-1) - - if not self.generated_manifest: - writemsg_level( - "Unable to generate manifest.", - level=logging.ERROR, noiselevel=-1) - failed = True - - if self.options.mode == "manifest": - if not failed and self.options.force and self.auto_assumed and \ - 'assume-digests' in self.repoman_settings.features: - # Show which digests were assumed despite the --force option - # being given. This output will already have been shown by - # digestgen() if assume-digests is not enabled, so only show - # it here if assume-digests is enabled. - pkgs = list(fetchlist_dict) - pkgs.sort() - portage.writemsg_stdout( - " digest.assumed %s" % - portage.output.colorize( - "WARN", str(len(self.auto_assumed)).rjust(18)) + "\n") - for cpv in pkgs: - fetchmap = fetchlist_dict[cpv] - pf = portage.catsplit(cpv)[1] - for distfile in sorted(fetchmap): - if distfile in self.auto_assumed: - portage.writemsg_stdout( - " %s::%s\n" % (pf, distfile)) - # continue, skip remaining main loop code - return True - elif failed: - sys.exit(1) - return False - - def _discard_dist_digests(self, checkdir, fetchlist_dict): - '''Discard DIST digests for files that exist in DISTDIR - - This method is intended to be called prior to digestgen, only for - manifest mode with the --force option, in order to discard DIST - digests that we intend to update. This is necessary because - digestgen never replaces existing digests, since otherwise it - would be too easy for ebuild developers to accidentally corrupt - existing DIST digests. - - @param checkdir: the directory to generate the Manifest in - @param fetchlist_dict: dictionary of files to fetch and/or include - in the manifest - ''' - portage._doebuild_manifest_exempt_depend += 1 - try: - distdir = self.repoman_settings['DISTDIR'] - mf = self.repoman_settings.repositories.get_repo_for_location( - os.path.dirname(os.path.dirname(checkdir))) - mf = mf.load_manifest( - checkdir, distdir, fetchlist_dict=fetchlist_dict) - mf.create( - requiredDistfiles=None, assumeDistHashesAlways=True) - for distfiles in fetchlist_dict.values(): - for distfile in distfiles: - if os.path.isfile(os.path.join(distdir, distfile)): - mf.fhashdict['DIST'].pop(distfile, None) - else: - self.auto_assumed.add(distfile) - mf.write() - finally: - portage._doebuild_manifest_exempt_depend -= 1 diff --git a/repoman/pym/repoman/modules/commit/repochecks.py b/repoman/pym/repoman/modules/commit/repochecks.py deleted file mode 100644 index bedbdaf34..000000000 --- a/repoman/pym/repoman/modules/commit/repochecks.py +++ /dev/null @@ -1,35 +0,0 @@ -# -*- coding:utf-8 -*- - -from __future__ import print_function, unicode_literals - -from portage.output import red - -from repoman.errors import err - - -def commit_check(repolevel, reposplit): - # Check if it's in $PORTDIR/$CATEGORY/$PN , otherwise bail if commiting. - # Reason for this is if they're trying to commit in just $FILESDIR/*, - # the Manifest needs updating. - # This check ensures that repoman knows where it is, - # and the manifest recommit is at least possible. - if repolevel not in [1, 2, 3]: - print(red("***") + ( - " Commit attempts *must* be from within a vcs checkout," - " category, or package directory.")) - print(red("***") + ( - " Attempting to commit from a packages files directory" - " will be blocked for instance.")) - print(red("***") + ( - " This is intended behaviour," - " to ensure the manifest is recommitted for a package.")) - print(red("***")) - err( - "Unable to identify level we're commiting from for %s" % - '/'.join(reposplit)) - - -def conflict_check(vcs_settings, options): - if vcs_settings.vcs: - conflicts = vcs_settings.status.detect_conflicts(options) - diff --git a/repoman/pym/repoman/modules/linechecks/__init__.py b/repoman/pym/repoman/modules/linechecks/__init__.py deleted file mode 100644 index 8b1378917..000000000 --- a/repoman/pym/repoman/modules/linechecks/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/repoman/pym/repoman/modules/linechecks/assignment/__init__.py b/repoman/pym/repoman/modules/linechecks/assignment/__init__.py deleted file mode 100644 index b95a25a74..000000000 --- a/repoman/pym/repoman/modules/linechecks/assignment/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Assignment plug-in module for repoman LineChecks. -Performs assignments checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'assignment', - 'description': doc, - 'provides':{ - 'assignment-check': { - 'name': "assignment", - 'sourcefile': "assignment", - 'class': "EbuildAssignment", - 'description': doc, - }, - 'eapi3-check': { - 'name': "eapi3assignment", - 'sourcefile': "assignment", - 'class': "Eapi3EbuildAssignment", - 'description': doc, - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/linechecks/assignment/assignment.py b/repoman/pym/repoman/modules/linechecks/assignment/assignment.py deleted file mode 100644 index 33bef8a08..000000000 --- a/repoman/pym/repoman/modules/linechecks/assignment/assignment.py +++ /dev/null @@ -1,38 +0,0 @@ - -import re - -from portage.eapi import eapi_supports_prefix, eapi_has_broot -from repoman.modules.linechecks.base import LineCheck - - -class EbuildAssignment(LineCheck): - """Ensure ebuilds don't assign to readonly variables.""" - - repoman_check_name = 'variable.readonly' - read_only_vars = 'A|CATEGORY|P|P[VNRF]|PVR|D|WORKDIR|FILESDIR|FEATURES|USE' - readonly_assignment = re.compile(r'^\s*(export\s+)?(%s)=' % read_only_vars) - - def check(self, num, line): - match = self.readonly_assignment.match(line) - e = None - if match is not None: - e = self.errors['READONLY_ASSIGNMENT_ERROR'] - return e - - -class Eapi3EbuildAssignment(EbuildAssignment): - """Ensure ebuilds don't assign to readonly EAPI 3-introduced variables.""" - - read_only_vars = 'ED|EPREFIX|EROOT' - readonly_assignment = re.compile(r'\s*(export\s+)?(%s)=' % read_only_vars) - - def check_eapi(self, eapi): - return eapi_supports_prefix(eapi) - -class Eapi7EbuildAssignment(EbuildAssignment): - """Ensure ebuilds don't assign to readonly EAPI 7-introduced variables.""" - - readonly_assignment = re.compile(r'\s*(export\s+)?BROOT=') - - def check_eapi(self, eapi): - return eapi_has_broot(eapi) diff --git a/repoman/pym/repoman/modules/linechecks/base.py b/repoman/pym/repoman/modules/linechecks/base.py deleted file mode 100644 index 4e3d6f0b4..000000000 --- a/repoman/pym/repoman/modules/linechecks/base.py +++ /dev/null @@ -1,101 +0,0 @@ - -import logging -import re - - -class LineCheck(object): - """Run a check on a line of an ebuild.""" - """A regular expression to determine whether to ignore the line""" - ignore_line = False - """True if lines containing nothing more than comments with optional - leading whitespace should be ignored""" - ignore_comment = True - - def __init__(self, errors): - self.errors = errors - - def new(self, pkg): - pass - - def check_eapi(self, eapi): - """Returns if check should be run in the given EAPI (default: True)""" - return True - - def check(self, num, line): - """Run the check on line and return error if there is one""" - if self.re.match(line): - return self.errors[self.error] - - def end(self): - pass - - -class InheritEclass(LineCheck): - """ - Base class for checking for missing inherits, as well as excess inherits. - - Args: - eclass: Set to the name of your eclass. - funcs: A tuple of functions that this eclass provides. - comprehensive: Is the list of functions complete? - exempt_eclasses: If these eclasses are inherited, disable the missing - inherit check. - """ - - def __init__( - self, eclass, eclass_eapi_functions, errors, funcs=None, comprehensive=False, - exempt_eclasses=None, ignore_missing=False, **kwargs): - self._eclass = eclass - self._comprehensive = comprehensive - self._exempt_eclasses = exempt_eclasses - self._ignore_missing = ignore_missing - self.errors = errors - inherit_re = eclass - self._eclass_eapi_functions = eclass_eapi_functions - self._inherit_re = re.compile( - r'^(\s*|.*[|&]\s*)\binherit\s(.*\s)?%s(\s|$)' % inherit_re) - # Match when the function is preceded only by leading whitespace, a - # shell operator such as (, {, |, ||, or &&, or optional variable - # setting(s). This prevents false positives in things like elog - # messages, as reported in bug #413285. - logging.debug("InheritEclass, eclass: %s, funcs: %s", eclass, funcs) - self._func_re = re.compile( - r'(^|[|&{(])\s*(\w+=.*)?\b(' + r'|'.join(funcs) + r')\b') - - def new(self, pkg): - self.repoman_check_name = 'inherit.missing' - # We can't use pkg.inherited because that tells us all the eclasses that - # have been inherited and not just the ones we inherit directly. - self._inherit = False - self._func_call = False - if self._exempt_eclasses is not None: - inherited = pkg.inherited - self._disabled = any(x in inherited for x in self._exempt_eclasses) - else: - self._disabled = False - self._eapi = pkg.eapi - - def check(self, num, line): - if not self._inherit: - self._inherit = self._inherit_re.match(line) - if not self._inherit: - if self._disabled or self._ignore_missing: - return - s = self._func_re.search(line) - if s is not None: - func_name = s.group(3) - eapi_func = self._eclass_eapi_functions.get(func_name) - if eapi_func is None or not eapi_func(self._eapi): - self._func_call = True - return ( - '%s.eclass is not inherited, ' - 'but "%s" found at line: %s' % - (self._eclass, func_name, '%d')) - elif not self._func_call: - self._func_call = self._func_re.search(line) - - def end(self): - if not self._disabled and self._comprehensive and self._inherit \ - and not self._func_call: - self.repoman_check_name = 'inherit.unused' - yield 'no function called from %s.eclass; please drop' % self._eclass diff --git a/repoman/pym/repoman/modules/linechecks/config.py b/repoman/pym/repoman/modules/linechecks/config.py deleted file mode 100644 index 6e4c5314e..000000000 --- a/repoman/pym/repoman/modules/linechecks/config.py +++ /dev/null @@ -1,118 +0,0 @@ -# -*- coding:utf-8 -*- -# repoman: Checks -# Copyright 2007-2017 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -"""This module contains functions used in Repoman to ascertain the quality -and correctness of an ebuild.""" - -from __future__ import unicode_literals - -import collections -import logging -import os -from copy import deepcopy - -from repoman._portage import portage -from repoman.config import load_config -from repoman import _not_installed - -# Avoid a circular import issue in py2.7 -portage.proxy.lazyimport.lazyimport(globals(), - 'portage.util:stack_lists', -) - - -def merge(dict1, dict2): - ''' Return a new dictionary by merging two dictionaries recursively. ''' - - result = deepcopy(dict1) - - for key, value in dict2.items(): - if isinstance(value, collections.Mapping): - result[key] = merge(result.get(key, {}), value) - else: - result[key] = deepcopy(dict2[key]) - - return result - - -class LineChecksConfig(object): - '''Holds our LineChecks configuration data and operation functions''' - - def __init__(self, repo_settings): - '''Class init - - @param repo_settings: RepoSettings instance - @param configpaths: ordered list of filepaths to load - ''' - self.repo_settings = repo_settings - self.infopaths = None - self.info_config = None - self._config = None - self.usex_supported_eapis = None - self.in_iuse_supported_eapis = None - self.get_libdir_supported_eapis = None - self.eclass_eapi_functions = {} - self.eclass_export_functions = None - self.eclass_info = {} - self.eclass_info_experimental_inherit = {} - self.errors = {} - self.set_infopaths() - self.load_checks_info() - - def set_infopaths(self): - if _not_installed: - cnfdir = os.path.realpath(os.path.join(os.path.dirname( - os.path.dirname(os.path.dirname(os.path.dirname( - os.path.dirname(__file__))))), 'cnf/linechecks')) - else: - cnfdir = os.path.join(portage.const.EPREFIX or '/', 'usr/share/repoman/linechecks') - repomanpaths = [os.path.join(cnfdir, _file_) for _file_ in os.listdir(cnfdir)] - logging.debug("LineChecksConfig; repomanpaths: %s", repomanpaths) - repopaths = [os.path.join(path, 'linechecks.yaml') for path in self.repo_settings.masters_list] - self.infopaths = repomanpaths + repopaths - logging.debug("LineChecksConfig; configpaths: %s", self.infopaths) - - def load_checks_info(self, infopaths=None): - '''load the config files in order - - @param infopaths: ordered list of filepaths to load - ''' - if infopaths: - self.infopaths = infopaths - elif not self.infopaths: - logging.error("LineChecksConfig; Error: No linechecks.yaml files defined") - - configs = load_config(self.infopaths, 'yaml', self.repo_settings.repoman_settings.valid_versions) - if configs == {}: - logging.error("LineChecksConfig: Failed to load a valid 'linechecks.yaml' file at paths: %s", self.infopaths) - return False - logging.debug("LineChecksConfig: linechecks.yaml configs: %s", configs) - self.info_config = configs - - self.errors = self.info_config['errors'] - self.usex_supported_eapis = self.info_config.get('usex_supported_eapis', []) - self.in_iuse_supported_eapis = self.info_config.get('in_iuse_supported_eapis', []) - self.eclass_info_experimental_inherit = self.info_config.get('eclass_info_experimental_inherit', []) - self.get_libdir_supported_eapis = self.in_iuse_supported_eapis - self.eclass_eapi_functions = { - "usex": lambda eapi: eapi not in self.usex_supported_eapis, - "in_iuse": lambda eapi: eapi not in self.in_iuse_supported_eapis, - "get_libdir": lambda eapi: eapi not in self.get_libdir_supported_eapis, - } - - # eclasses that export ${ECLASS}_src_(compile|configure|install) - self.eclass_export_functions = self.info_config.get('eclass_export_functions', []) - - self.eclass_info_experimental_inherit = self.info_config.get('eclass_info_experimental_inherit', {}) - # These are "eclasses are the whole ebuild" type thing. - try: - self.eclass_info_experimental_inherit['eutils']['exempt_eclasses'] = self.eclass_export_functions - except KeyError: - pass - try: - self.eclass_info_experimental_inherit['multilib']['exempt_eclasses'] = self.eclass_export_functions + [ - 'autotools', 'libtool', 'multilib-minimal'] - except KeyError: - pass diff --git a/repoman/pym/repoman/modules/linechecks/controller.py b/repoman/pym/repoman/modules/linechecks/controller.py deleted file mode 100644 index 7082a5d02..000000000 --- a/repoman/pym/repoman/modules/linechecks/controller.py +++ /dev/null @@ -1,145 +0,0 @@ - -import logging -import operator -import os -import re - -from repoman.modules.linechecks.base import InheritEclass -from repoman.modules.linechecks.config import LineChecksConfig -from repoman._portage import portage - -# Avoid a circular import issue in py2.7 -portage.proxy.lazyimport.lazyimport(globals(), - 'portage.module:Modules', -) - -MODULES_PATH = os.path.dirname(__file__) -# initial development debug info -logging.debug("LineChecks module path: %s", MODULES_PATH) - - -class LineCheckController(object): - '''Initializes and runs the LineCheck checks''' - - def __init__(self, repo_settings, linechecks): - '''Class init - - @param repo_settings: RepoSettings instance - ''' - self.repo_settings = repo_settings - self.linechecks = linechecks - self.config = LineChecksConfig(repo_settings) - - self.controller = Modules(path=MODULES_PATH, namepath="repoman.modules.linechecks") - logging.debug("LineCheckController; module_names: %s", self.controller.module_names) - - self._constant_checks = None - - self._here_doc_re = re.compile(r'.*<<[-]?(\w+)\s*(>\s*\S+\s*)?$') - self._ignore_comment_re = re.compile(r'^\s*#') - self._continuation_re = re.compile(r'(\\)*$') - - def checks_init(self, experimental_inherit=False): - '''Initialize the main variables - - @param experimental_inherit boolean - ''' - if not experimental_inherit: - # Emulate the old eprefixify.defined and inherit.autotools checks. - self._eclass_info = self.config.eclass_info - else: - self._eclass_info = self.config.eclass_info_experimental_inherit - - self._constant_checks = [] - logging.debug("LineCheckController; modules: %s", self.linechecks) - # Add in the pluggable modules - for mod in self.linechecks: - mod_class = self.controller.get_class(mod) - logging.debug("LineCheckController; module_name: %s, class: %s", mod, mod_class.__name__) - self._constant_checks.append(mod_class(self.config.errors)) - # Add in the InheritEclass checks - logging.debug("LineCheckController; eclass_info.items(): %s", list(self.config.eclass_info)) - for k, kwargs in self.config.eclass_info.items(): - logging.debug("LineCheckController; k: %s, kwargs: %s", k, kwargs) - self._constant_checks.append( - InheritEclass( - k, - self.config.eclass_eapi_functions, - self.config.errors, - **kwargs - ) - ) - - - def run_checks(self, contents, pkg): - '''Run the configured linechecks - - @param contents: the ebjuild contents to check - @param pkg: the package being checked - ''' - if self._constant_checks is None: - self.checks_init() - checks = self._constant_checks - here_doc_delim = None - multiline = None - - for lc in checks: - lc.new(pkg) - - multinum = 0 - for num, line in enumerate(contents): - - # Check if we're inside a here-document. - if here_doc_delim is not None: - if here_doc_delim.match(line): - here_doc_delim = None - if here_doc_delim is None: - here_doc = self._here_doc_re.match(line) - if here_doc is not None: - here_doc_delim = re.compile(r'^\s*%s$' % here_doc.group(1)) - if here_doc_delim is not None: - continue - - # Unroll multiline escaped strings so that we can check things: - # inherit foo bar \ - # moo \ - # cow - # This will merge these lines like so: - # inherit foo bar moo cow - # A line ending with an even number of backslashes does not count, - # because the last backslash is escaped. Therefore, search for an - # odd number of backslashes. - line_escaped = operator.sub(*self._continuation_re.search(line).span()) % 2 == 1 - if multiline: - # Chop off the \ and \n bytes from the previous line. - multiline = multiline[:-2] + line - if not line_escaped: - line = multiline - num = multinum - multiline = None - else: - continue - else: - if line_escaped: - multinum = num - multiline = line - continue - - if not line.endswith("#nowarn\n"): - # Finally we have a full line to parse. - is_comment = self._ignore_comment_re.match(line) is not None - for lc in checks: - if is_comment and lc.ignore_comment: - continue - if lc.check_eapi(pkg.eapi): - ignore = lc.ignore_line - if not ignore or not ignore.match(line): - e = lc.check(num, line) - if e: - yield lc.repoman_check_name, e % (num + 1) - - for lc in checks: - i = lc.end() - if i is not None: - for e in i: - yield lc.repoman_check_name, e diff --git a/repoman/pym/repoman/modules/linechecks/depend/__init__.py b/repoman/pym/repoman/modules/linechecks/depend/__init__.py deleted file mode 100644 index 2ea95347e..000000000 --- a/repoman/pym/repoman/modules/linechecks/depend/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Depend plug-in module for repoman LineChecks. -Performs dependency checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'depend', - 'description': doc, - 'provides':{ - 'implicit-check': { - 'name': "implicitdepend", - 'sourcefile': "implicit", - 'class': "ImplicitRuntimeDeps", - 'description': doc, - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/linechecks/depend/implicit.py b/repoman/pym/repoman/modules/linechecks/depend/implicit.py deleted file mode 100644 index f7b458b68..000000000 --- a/repoman/pym/repoman/modules/linechecks/depend/implicit.py +++ /dev/null @@ -1,39 +0,0 @@ - -import re - -from portage.eapi import eapi_has_implicit_rdepend -from repoman.modules.linechecks.base import LineCheck - - -class ImplicitRuntimeDeps(LineCheck): - """ - Detect the case where DEPEND is set and RDEPEND is unset in the ebuild, - since this triggers implicit RDEPEND=$DEPEND assignment (prior to EAPI 4). - """ - - repoman_check_name = 'RDEPEND.implicit' - _assignment_re = re.compile(r'^\s*(R?DEPEND)\+?=') - - def new(self, pkg): - self._rdepend = False - self._depend = False - - def check_eapi(self, eapi): - # Beginning with EAPI 4, there is no - # implicit RDEPEND=$DEPEND assignment - # to be concerned with. - return eapi_has_implicit_rdepend(eapi) - - def check(self, num, line): - if not self._rdepend: - m = self._assignment_re.match(line) - if m is None: - pass - elif m.group(1) == "RDEPEND": - self._rdepend = True - elif m.group(1) == "DEPEND": - self._depend = True - - def end(self): - if self._depend and not self._rdepend: - yield 'RDEPEND is not explicitly assigned' diff --git a/repoman/pym/repoman/modules/linechecks/deprecated/__init__.py b/repoman/pym/repoman/modules/linechecks/deprecated/__init__.py deleted file mode 100644 index 258683345..000000000 --- a/repoman/pym/repoman/modules/linechecks/deprecated/__init__.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Deprecated plug-in module for repoman LineChecks. -Performs miscelaneous deprecation checks on ebuilds not covered by -specialty modules.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'deprecated', - 'description': doc, - 'provides':{ - 'useq-check': { - 'name': "useq", - 'sourcefile': "deprecated", - 'class': "DeprecatedUseq", - 'description': doc, - }, - 'hasq-check': { - 'name': "hasq", - 'sourcefile': "deprecated", - 'class': "DeprecatedHasq", - 'description': doc, - }, - 'preserve-check': { - 'name': "preservelib", - 'sourcefile': "deprecated", - 'class': "PreserveOldLib", - 'description': doc, - }, - 'bindnow-check': { - 'name': "bindnow", - 'sourcefile': "deprecated", - 'class': "DeprecatedBindnowFlags", - 'description': doc, - }, - 'inherit-check': { - 'name': "inherit", - 'sourcefile': "inherit", - 'class': "InheritDeprecated", - 'description': doc, - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/linechecks/deprecated/deprecated.py b/repoman/pym/repoman/modules/linechecks/deprecated/deprecated.py deleted file mode 100644 index b33852e74..000000000 --- a/repoman/pym/repoman/modules/linechecks/deprecated/deprecated.py +++ /dev/null @@ -1,32 +0,0 @@ - -import re - -from repoman.modules.linechecks.base import LineCheck - - -class DeprecatedUseq(LineCheck): - """Checks for use of the deprecated useq function""" - repoman_check_name = 'ebuild.minorsyn' - re = re.compile(r'(^|.*\b)useq\b') - error = 'USEQ_ERROR' - - -class DeprecatedHasq(LineCheck): - """Checks for use of the deprecated hasq function""" - repoman_check_name = 'ebuild.minorsyn' - re = re.compile(r'(^|.*\b)hasq\b') - error = 'HASQ_ERROR' - - -class PreserveOldLib(LineCheck): - """Check for calls to the deprecated preserve_old_lib function.""" - repoman_check_name = 'ebuild.minorsyn' - re = re.compile(r'.*preserve_old_lib') - error = 'PRESERVE_OLD_LIB' - - -class DeprecatedBindnowFlags(LineCheck): - """Check for calls to the deprecated bindnow-flags function.""" - repoman_check_name = 'ebuild.minorsyn' - re = re.compile(r'.*\$\(bindnow-flags\)') - error = 'DEPRECATED_BINDNOW_FLAGS' diff --git a/repoman/pym/repoman/modules/linechecks/deprecated/inherit.py b/repoman/pym/repoman/modules/linechecks/deprecated/inherit.py deleted file mode 100644 index 77ad4f625..000000000 --- a/repoman/pym/repoman/modules/linechecks/deprecated/inherit.py +++ /dev/null @@ -1,69 +0,0 @@ - -import re - -from repoman.modules.linechecks.base import LineCheck - - -class InheritDeprecated(LineCheck): - """Check if ebuild directly or indirectly inherits a deprecated eclass.""" - - repoman_check_name = 'inherit.deprecated' - - # deprecated eclass : new eclass (False if no new eclass) - deprecated_eclasses = { - "base": False, - "bash-completion": "bash-completion-r1", - "boost-utils": False, - "clutter": "gnome2", - "confutils": False, - "distutils": "distutils-r1", - "epatch": "(eapply since EAPI 6)", - "games": False, - "gems": "ruby-fakegem", - "gpe": False, - "gst-plugins-bad": "gstreamer", - "gst-plugins-base": "gstreamer", - "gst-plugins-good": "gstreamer", - "gst-plugins-ugly": "gstreamer", - "gst-plugins10": "gstreamer", - "ltprune": False, - "mono": "mono-env", - "python": "python-r1 / python-single-r1 / python-any-r1", - "ruby": "ruby-ng", - "versionator": "eapi7-ver (built-in since EAPI 7)", - "x-modular": "xorg-2", - } - - _inherit_re = re.compile(r'^\s*inherit\s(.*)$') - - def new(self, pkg): - self._errors = [] - - def check(self, num, line): - direct_inherits = None - m = self._inherit_re.match(line) - if m is not None: - direct_inherits = m.group(1) - if direct_inherits: - direct_inherits = direct_inherits.split() - - if not direct_inherits: - return - - for eclass in direct_inherits: - replacement = self.deprecated_eclasses.get(eclass) - if replacement is None: - pass - elif replacement is False: - self._errors.append( - "please migrate from " - "'%s' (no replacement) on line: %d" % (eclass, num + 1)) - else: - self._errors.append( - "please migrate from " - "'%s' to '%s' on line: %d" % (eclass, replacement, num + 1)) - - def end(self): - for error in self._errors: - yield error - del self._errors diff --git a/repoman/pym/repoman/modules/linechecks/do/__init__.py b/repoman/pym/repoman/modules/linechecks/do/__init__.py deleted file mode 100644 index dc5af701c..000000000 --- a/repoman/pym/repoman/modules/linechecks/do/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Do plug-in module for repoman LineChecks. -Performs do* checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'do', - 'description': doc, - 'provides':{ - 'nonrelative-check': { - 'name': "dosym", - 'sourcefile': "dosym", - 'class': "EbuildNonRelativeDosym", - 'description': doc, - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/linechecks/do/dosym.py b/repoman/pym/repoman/modules/linechecks/do/dosym.py deleted file mode 100644 index bab4dad03..000000000 --- a/repoman/pym/repoman/modules/linechecks/do/dosym.py +++ /dev/null @@ -1,16 +0,0 @@ - -import re - -from repoman.modules.linechecks.base import LineCheck - - -class EbuildNonRelativeDosym(LineCheck): - """Check ebuild for dosym using absolute paths instead of relative.""" - repoman_check_name = 'ebuild.absdosym' - regex = re.compile( - r'^\s*dosym\s+["\']?(/(bin|etc|lib|opt|sbin|srv|usr|var)\S*)') - - def check(self, num, line): - match = self.regex.match(line) - if match: - return "dosym '%s'... could use relative path" % (match.group(1), ) + " on line: %d" diff --git a/repoman/pym/repoman/modules/linechecks/eapi/__init__.py b/repoman/pym/repoman/modules/linechecks/eapi/__init__.py deleted file mode 100644 index 31993df20..000000000 --- a/repoman/pym/repoman/modules/linechecks/eapi/__init__.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Eapi plug-in module for repoman LineChecks. -Performs eapi dependant checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'eapi', - 'description': doc, - 'provides':{ - 'definition-check': { - 'name': "definition", - 'sourcefile': "definition", - 'class': "EapiDefinition", - 'description': doc, - }, - 'srcprepare-check': { - 'name': "srcprepare", - 'sourcefile': "checks", - 'class': "UndefinedSrcPrepareSrcConfigurePhases", - 'description': doc, - }, - 'eapi3deprecated-check': { - 'name': "eapi3deprecated", - 'sourcefile': "checks", - 'class': "Eapi3DeprecatedFuncs", - 'description': doc, - }, - 'pkgpretend-check': { - 'name': "pkgpretend", - 'sourcefile': "checks", - 'class': "UndefinedPkgPretendPhase", - 'description': doc, - }, - 'eapi4incompatible-check': { - 'name': "eapi4incompatible", - 'sourcefile': "checks", - 'class': "Eapi4IncompatibleFuncs", - 'description': doc, - }, - 'eapi4gonevars-check': { - 'name': "eapi4gonevars", - 'sourcefile': "checks", - 'class': "Eapi4GoneVars", - 'description': doc, - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/linechecks/eapi/checks.py b/repoman/pym/repoman/modules/linechecks/eapi/checks.py deleted file mode 100644 index de899c061..000000000 --- a/repoman/pym/repoman/modules/linechecks/eapi/checks.py +++ /dev/null @@ -1,83 +0,0 @@ - -import re - -from portage.eapi import ( - eapi_has_src_prepare_and_src_configure, eapi_has_dosed_dohard, - eapi_exports_AA, eapi_has_pkg_pretend) -from repoman.modules.linechecks.base import LineCheck - - -# EAPI <2 checks -class UndefinedSrcPrepareSrcConfigurePhases(LineCheck): - repoman_check_name = 'EAPI.incompatible' - src_configprepare_re = re.compile(r'\s*(src_configure|src_prepare)\s*\(\)') - - def check_eapi(self, eapi): - return not eapi_has_src_prepare_and_src_configure(eapi) - - def check(self, num, line): - m = self.src_configprepare_re.match(line) - if m is not None: - return ("'%s'" % m.group(1)) + \ - " phase is not defined in EAPI < 2 on line: %d" - - -# EAPI-3 checks -class Eapi3DeprecatedFuncs(LineCheck): - repoman_check_name = 'EAPI.deprecated' - deprecated_commands_re = re.compile(r'^\s*(check_license)\b') - - def check_eapi(self, eapi): - return eapi not in ('0', '1', '2') - - def check(self, num, line): - m = self.deprecated_commands_re.match(line) - if m is not None: - return ("'%s'" % m.group(1)) + \ - " has been deprecated in EAPI=3 on line: %d" - - -# EAPI <4 checks -class UndefinedPkgPretendPhase(LineCheck): - repoman_check_name = 'EAPI.incompatible' - pkg_pretend_re = re.compile(r'\s*(pkg_pretend)\s*\(\)') - - def check_eapi(self, eapi): - return not eapi_has_pkg_pretend(eapi) - - def check(self, num, line): - m = self.pkg_pretend_re.match(line) - if m is not None: - return ("'%s'" % m.group(1)) + \ - " phase is not defined in EAPI < 4 on line: %d" - - -# EAPI-4 checks -class Eapi4IncompatibleFuncs(LineCheck): - repoman_check_name = 'EAPI.incompatible' - banned_commands_re = re.compile(r'^\s*(dosed|dohard)') - - def check_eapi(self, eapi): - return not eapi_has_dosed_dohard(eapi) - - def check(self, num, line): - m = self.banned_commands_re.match(line) - if m is not None: - return ("'%s'" % m.group(1)) + \ - " has been banned in EAPI=4 on line: %d" - - -class Eapi4GoneVars(LineCheck): - repoman_check_name = 'EAPI.incompatible' - undefined_vars_re = re.compile( - r'.*\$(\{(AA|KV|EMERGE_FROM)\}|(AA|KV|EMERGE_FROM))') - - def check_eapi(self, eapi): - # AA, KV, and EMERGE_FROM should not be referenced in EAPI 4 or later. - return not eapi_exports_AA(eapi) - - def check(self, num, line): - m = self.undefined_vars_re.match(line) - if m is not None: - return ("variable '$%s'" % m.group(1)) + \ - " is gone in EAPI=4 on line: %d" diff --git a/repoman/pym/repoman/modules/linechecks/eapi/definition.py b/repoman/pym/repoman/modules/linechecks/eapi/definition.py deleted file mode 100644 index 410bad1c7..000000000 --- a/repoman/pym/repoman/modules/linechecks/eapi/definition.py +++ /dev/null @@ -1,36 +0,0 @@ - -from repoman.modules.linechecks.base import LineCheck -from repoman._portage import portage - - -class EapiDefinition(LineCheck): - """ - Check that EAPI assignment conforms to PMS section 7.3.1 - (first non-comment, non-blank line). - """ - repoman_check_name = 'EAPI.definition' - ignore_comment = True - _eapi_re = portage._pms_eapi_re - - def new(self, pkg): - self._cached_eapi = pkg.eapi - self._parsed_eapi = None - self._eapi_line_num = None - - def check(self, num, line): - if self._eapi_line_num is None and line.strip(): - self._eapi_line_num = num + 1 - m = self._eapi_re.match(line) - if m is not None: - self._parsed_eapi = m.group(2) - - def end(self): - if self._parsed_eapi is None: - if self._cached_eapi != "0": - yield "valid EAPI assignment must occur on or before line: %s" % \ - self._eapi_line_num - elif self._parsed_eapi != self._cached_eapi: - yield ( - "bash returned EAPI '%s' which does not match " - "assignment on line: %s" % - (self._cached_eapi, self._eapi_line_num)) diff --git a/repoman/pym/repoman/modules/linechecks/emake/__init__.py b/repoman/pym/repoman/modules/linechecks/emake/__init__.py deleted file mode 100644 index cecdc5fcf..000000000 --- a/repoman/pym/repoman/modules/linechecks/emake/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Emake plug-in module for repoman LineChecks. -Performs emake checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'do', - 'description': doc, - 'provides':{ - 'paralleldisabled-check': { - 'name': "paralleldisabled", - 'sourcefile': "emake", - 'class': "EMakeParallelDisabledViaMAKEOPTS", - 'description': doc, - }, - 'autodefault-check': { - 'name': "autodefault", - 'sourcefile': "emake", - 'class': "WantAutoDefaultValue", - 'description': doc, - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/linechecks/emake/emake.py b/repoman/pym/repoman/modules/linechecks/emake/emake.py deleted file mode 100644 index e1e3e638e..000000000 --- a/repoman/pym/repoman/modules/linechecks/emake/emake.py +++ /dev/null @@ -1,23 +0,0 @@ - -import re - -from repoman.modules.linechecks.base import LineCheck - - -class EMakeParallelDisabledViaMAKEOPTS(LineCheck): - """Check for MAKEOPTS=-j1 that disables parallelization.""" - repoman_check_name = 'upstream.workaround' - re = re.compile(r'^\s*MAKEOPTS=(\'|")?.*-j\s*1\b') - error = 'EMAKE_PARALLEL_DISABLED_VIA_MAKEOPTS' - - -class WantAutoDefaultValue(LineCheck): - """Check setting WANT_AUTO* to latest (default value).""" - repoman_check_name = 'ebuild.minorsyn' - _re = re.compile(r'^WANT_AUTO(CONF|MAKE)=(\'|")?latest') - - def check(self, num, line): - m = self._re.match(line) - if m is not None: - return 'WANT_AUTO' + m.group(1) + \ - ' redundantly set to default value "latest" on line: %d' diff --git a/repoman/pym/repoman/modules/linechecks/gentoo_header/__init__.py b/repoman/pym/repoman/modules/linechecks/gentoo_header/__init__.py deleted file mode 100644 index 205cc32c2..000000000 --- a/repoman/pym/repoman/modules/linechecks/gentoo_header/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Gentoo-header plug-in module for repoman LineChecks. -Performs header checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'do', - 'description': doc, - 'provides':{ - 'header-check': { - 'name': "gentooheader", - 'sourcefile': "header", - 'class': "EbuildHeader", - 'description': doc, - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/linechecks/gentoo_header/header.py b/repoman/pym/repoman/modules/linechecks/gentoo_header/header.py deleted file mode 100644 index 4b75fc4b5..000000000 --- a/repoman/pym/repoman/modules/linechecks/gentoo_header/header.py +++ /dev/null @@ -1,49 +0,0 @@ - -import re -import time - -from repoman.modules.linechecks.base import LineCheck - - -class EbuildHeader(LineCheck): - """Ensure ebuilds have proper headers - Copyright header errors - CVS header errors - License header errors - - Args: - modification_year - Year the ebuild was last modified - """ - - repoman_check_name = 'ebuild.badheader' - - gentoo_copyright = r'^# Copyright ((1999|2\d\d\d)-)?%s Gentoo Foundation$' - gentoo_license = ( - '# Distributed under the terms' - ' of the GNU General Public License v2') - id_header_re = re.compile(r'.*\$(Id|Header)(:.*)?\$.*') - blank_line_re = re.compile(r'^$') - ignore_comment = False - - def new(self, pkg): - if pkg.mtime is None: - self.modification_year = r'2\d\d\d' - else: - self.modification_year = str(time.gmtime(pkg.mtime)[0]) - self.gentoo_copyright_re = re.compile( - self.gentoo_copyright % self.modification_year) - - def check(self, num, line): - if num > 2: - return - elif num == 0: - if not self.gentoo_copyright_re.match(line): - return self.errors['COPYRIGHT_ERROR'] - elif num == 1 and line.rstrip('\n') != self.gentoo_license: - return self.errors['LICENSE_ERROR'] - elif num == 2 and self.id_header_re.match(line): - return self.errors['ID_HEADER_ERROR'] - elif num == 2 and not self.blank_line_re.match(line): - return self.errors['NO_BLANK_LINE_ERROR'] - - diff --git a/repoman/pym/repoman/modules/linechecks/helpers/__init__.py b/repoman/pym/repoman/modules/linechecks/helpers/__init__.py deleted file mode 100644 index 0f41f0136..000000000 --- a/repoman/pym/repoman/modules/linechecks/helpers/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Helpers plug-in module for repoman LineChecks. -Performs variable helpers checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'do', - 'description': doc, - 'provides':{ - 'nooffset-check': { - 'name': "nooffset", - 'sourcefile': "offset", - 'class': "NoOffsetWithHelpers", - 'description': doc, - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/linechecks/helpers/offset.py b/repoman/pym/repoman/modules/linechecks/helpers/offset.py deleted file mode 100644 index 5d7624a68..000000000 --- a/repoman/pym/repoman/modules/linechecks/helpers/offset.py +++ /dev/null @@ -1,22 +0,0 @@ - -import re - -from repoman.modules.linechecks.base import LineCheck - - -class NoOffsetWithHelpers(LineCheck): - """ Check that the image location, the alternate root offset, and the - offset prefix (D, ROOT, ED, EROOT and EPREFIX) are not used with - helpers """ - - repoman_check_name = 'variable.usedwithhelpers' - # Ignore matches in quoted strings like this: - # elog "installed into ${ROOT}usr/share/php5/apc/." - _install_funcs = ( - 'docinto|do(compress|dir|hard)' - '|exeinto|fowners|fperms|insinto|into') - _quoted_vars = 'D|ROOT|ED|EROOT|EPREFIX' - re = re.compile( - r'^[^#"\']*\b(%s)\s+"?\$\{?(%s)\b.*' % - (_install_funcs, _quoted_vars)) - error = 'NO_OFFSET_WITH_HELPERS' diff --git a/repoman/pym/repoman/modules/linechecks/nested/__init__.py b/repoman/pym/repoman/modules/linechecks/nested/__init__.py deleted file mode 100644 index 6c8a70a52..000000000 --- a/repoman/pym/repoman/modules/linechecks/nested/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Nested plug-in module for repoman LineChecks. -Performs nested subshell checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'do', - 'description': doc, - 'provides':{ - 'nesteddie-check': { - 'name': "nesteddie", - 'sourcefile': "nested", - 'class': "EbuildNestedDie", - 'description': doc, - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/linechecks/nested/nested.py b/repoman/pym/repoman/modules/linechecks/nested/nested.py deleted file mode 100644 index 06b272772..000000000 --- a/repoman/pym/repoman/modules/linechecks/nested/nested.py +++ /dev/null @@ -1,15 +0,0 @@ - -import re - -from repoman.modules.linechecks.base import LineCheck - - -class EbuildNestedDie(LineCheck): - """Check ebuild for nested die statements (die statements in subshells)""" - - repoman_check_name = 'ebuild.nesteddie' - nesteddie_re = re.compile(r'^[^#]*\s\(\s[^)]*\bdie\b') - - def check(self, num, line): - if self.nesteddie_re.match(line): - return self.errors['NESTED_DIE_ERROR'] diff --git a/repoman/pym/repoman/modules/linechecks/nested/nesteddie.py b/repoman/pym/repoman/modules/linechecks/nested/nesteddie.py deleted file mode 100644 index 6c1e4be9f..000000000 --- a/repoman/pym/repoman/modules/linechecks/nested/nesteddie.py +++ /dev/null @@ -1,11 +0,0 @@ - - -class EbuildNestedDie(LineCheck): - """Check ebuild for nested die statements (die statements in subshells)""" - - repoman_check_name = 'ebuild.nesteddie' - nesteddie_re = re.compile(r'^[^#]*\s\(\s[^)]*\bdie\b') - - def check(self, num, line): - if self.nesteddie_re.match(line): - return errors.NESTED_DIE_ERROR diff --git a/repoman/pym/repoman/modules/linechecks/patches/__init__.py b/repoman/pym/repoman/modules/linechecks/patches/__init__.py deleted file mode 100644 index 4fe0fcf5e..000000000 --- a/repoman/pym/repoman/modules/linechecks/patches/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Patches plug-in module for repoman LineChecks. -Performs PATCHES variable checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'do', - 'description': doc, - 'provides':{ - 'patches-check': { - 'name': "patches", - 'sourcefile': "patches", - 'class': "EbuildPatches", - 'description': doc, - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/linechecks/patches/patches.py b/repoman/pym/repoman/modules/linechecks/patches/patches.py deleted file mode 100644 index 63651cd7c..000000000 --- a/repoman/pym/repoman/modules/linechecks/patches/patches.py +++ /dev/null @@ -1,16 +0,0 @@ - - -import re - -from repoman.modules.linechecks.base import LineCheck - - -class EbuildPatches(LineCheck): - """Ensure ebuilds use bash arrays for PATCHES to ensure white space safety""" - repoman_check_name = 'ebuild.patches' - re = re.compile(r'^\s*PATCHES=[^\(]') - error = 'PATCHES_ERROR' - - def check_eapi(self, eapi): - return eapi in ("0", "1", "2", "3", "4", "4-python", - "4-slot-abi", "5", "5-hdepend", "5-progress") diff --git a/repoman/pym/repoman/modules/linechecks/phases/__init__.py b/repoman/pym/repoman/modules/linechecks/phases/__init__.py deleted file mode 100644 index b3228c0f7..000000000 --- a/repoman/pym/repoman/modules/linechecks/phases/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Phases plug-in module for repoman LineChecks. -Performs phase dependant checks on ebuilds using a PhaseCheck base class. -""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'do', - 'description': doc, - 'provides':{ - 'emakeparallel-check': { - 'name': "emakeparallel", - 'sourcefile': "phase", - 'class': "EMakeParallelDisabled", - 'description': doc, - }, - 'srccompileeconf-check': { - 'name': "srccompileeconf", - 'sourcefile': "phase", - 'class': "SrcCompileEconf", - 'description': doc, - }, - 'srcunpackpatches-check': { - 'name': "srcunpackpatches", - 'sourcefile': "phase", - 'class': "SrcUnpackPatches", - 'description': doc, - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/linechecks/phases/phase.py b/repoman/pym/repoman/modules/linechecks/phases/phase.py deleted file mode 100644 index acc3a1e1d..000000000 --- a/repoman/pym/repoman/modules/linechecks/phases/phase.py +++ /dev/null @@ -1,71 +0,0 @@ - -import re - -from portage.eapi import eapi_has_src_prepare_and_src_configure -from repoman.modules.linechecks.base import LineCheck - - -class PhaseCheck(LineCheck): - """ basic class for function detection """ - - func_end_re = re.compile(r'^\}$') - phases_re = re.compile('(%s)' % '|'.join(( - 'pkg_pretend', 'pkg_setup', 'src_unpack', 'src_prepare', - 'src_configure', 'src_compile', 'src_test', 'src_install', - 'pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm', - 'pkg_config'))) - in_phase = '' - - def check(self, num, line): - m = self.phases_re.match(line) - if m is not None: - self.in_phase = m.group(1) - if self.in_phase != '' and self.func_end_re.match(line) is not None: - self.in_phase = '' - - return self.phase_check(num, line) - - def phase_check(self, num, line): - """ override this function for your checks """ - pass - - -class EMakeParallelDisabled(PhaseCheck): - """Check for emake -j1 calls which disable parallelization.""" - repoman_check_name = 'upstream.workaround' - re = re.compile(r'^\s*emake\s+.*-j\s*1\b') - - def phase_check(self, num, line): - if self.in_phase == 'src_compile' or self.in_phase == 'src_install': - if self.re.match(line): - return self.errors['EMAKE_PARALLEL_DISABLED'] - - -class SrcCompileEconf(PhaseCheck): - repoman_check_name = 'ebuild.minorsyn' - configure_re = re.compile(r'\s(econf|./configure)') - - def check_eapi(self, eapi): - return eapi_has_src_prepare_and_src_configure(eapi) - - def phase_check(self, num, line): - if self.in_phase == 'src_compile': - m = self.configure_re.match(line) - if m is not None: - return ("'%s'" % m.group(1)) + \ - " call should be moved to src_configure from line: %d" - - -class SrcUnpackPatches(PhaseCheck): - repoman_check_name = 'ebuild.minorsyn' - src_prepare_tools_re = re.compile(r'\s(e?patch|sed)\s') - - def check_eapi(self, eapi): - return eapi_has_src_prepare_and_src_configure(eapi) - - def phase_check(self, num, line): - if self.in_phase == 'src_unpack': - m = self.src_prepare_tools_re.search(line) - if m is not None: - return ("'%s'" % m.group(1)) + \ - " call should be moved to src_prepare from line: %d" diff --git a/repoman/pym/repoman/modules/linechecks/portage/__init__.py b/repoman/pym/repoman/modules/linechecks/portage/__init__.py deleted file mode 100644 index b9a84b507..000000000 --- a/repoman/pym/repoman/modules/linechecks/portage/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Portage plug-in module for repoman LineChecks. -Performs checks for internal portage variable usage in ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'do', - 'description': doc, - 'provides':{ - 'internal-check': { - 'name': "portageinternal", - 'sourcefile': "internal", - 'class': "PortageInternal", - 'description': doc, - }, - 'portageinternalvariableassignment-check': { - 'name': "portageinternalvariableassignment", - 'sourcefile': "internal", - 'class': "PortageInternalVariableAssignment", - 'description': doc, - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/linechecks/portage/internal.py b/repoman/pym/repoman/modules/linechecks/portage/internal.py deleted file mode 100644 index 869337221..000000000 --- a/repoman/pym/repoman/modules/linechecks/portage/internal.py +++ /dev/null @@ -1,37 +0,0 @@ - -import re - -from repoman.modules.linechecks.base import LineCheck - - -class PortageInternal(LineCheck): - repoman_check_name = 'portage.internal' - ignore_comment = True - # Match when the command is preceded only by leading whitespace or a shell - # operator such as (, {, |, ||, or &&. This prevents false positives in - # things like elog messages, as reported in bug #413285. - - internal_portage_func_or_var = ( - 'ecompress|ecompressdir|env-update|prepall|prepalldocs|preplib') - re = re.compile( - r'^(\s*|.*[|&{(]+\s*)\b(%s)\b' % internal_portage_func_or_var) - - def check(self, num, line): - """Run the check on line and return error if there is one""" - m = self.re.match(line) - if m is not None: - return ("'%s'" % m.group(2)) + " called on line: %d" - - -class PortageInternalVariableAssignment(LineCheck): - repoman_check_name = 'portage.internal' - internal_assignment = re.compile( - r'\s*(export\s+)?(EXTRA_ECONF|EXTRA_EMAKE)\+?=') - - def check(self, num, line): - match = self.internal_assignment.match(line) - e = None - if match is not None: - e = 'Assignment to variable %s' % match.group(2) - e += ' on line: %d' - return e diff --git a/repoman/pym/repoman/modules/linechecks/quotes/__init__.py b/repoman/pym/repoman/modules/linechecks/quotes/__init__.py deleted file mode 100644 index 894ef58d2..000000000 --- a/repoman/pym/repoman/modules/linechecks/quotes/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Nested plug-in module for repoman LineChecks. -Performs nested subshell checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'do', - 'description': doc, - 'provides':{ - 'quote-check': { - 'name': "quote", - 'sourcefile': "quotes", - 'class': "EbuildQuote", - 'description': doc, - }, - 'quoteda-check': { - 'name': "quoteda", - 'sourcefile': "quoteda", - 'class': "EbuildQuotedA", - 'description': doc, - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/linechecks/quotes/quoteda.py b/repoman/pym/repoman/modules/linechecks/quotes/quoteda.py deleted file mode 100644 index 7fd9ba797..000000000 --- a/repoman/pym/repoman/modules/linechecks/quotes/quoteda.py +++ /dev/null @@ -1,16 +0,0 @@ - -import re - -from repoman.modules.linechecks.base import LineCheck - - -class EbuildQuotedA(LineCheck): - """Ensure ebuilds have no quoting around ${A}""" - - repoman_check_name = 'ebuild.minorsyn' - a_quoted = re.compile(r'.*\"\$(\{A\}|A)\"') - - def check(self, num, line): - match = self.a_quoted.match(line) - if match: - return "Quoted \"${A}\" on line: %d" diff --git a/repoman/pym/repoman/modules/linechecks/quotes/quotes.py b/repoman/pym/repoman/modules/linechecks/quotes/quotes.py deleted file mode 100644 index e5ea4d0ca..000000000 --- a/repoman/pym/repoman/modules/linechecks/quotes/quotes.py +++ /dev/null @@ -1,87 +0,0 @@ - -import re - -from repoman.modules.linechecks.base import LineCheck - - -class EbuildQuote(LineCheck): - """Ensure ebuilds have valid quoting around things like D,FILESDIR, etc...""" - - repoman_check_name = 'ebuild.minorsyn' - _message_commands = [ - "die", "echo", "eerror", "einfo", "elog", "eqawarn", "ewarn"] - _message_re = re.compile( - r'\s(' + "|".join(_message_commands) + r')\s+"[^"]*"\s*$') - _ignored_commands = ["local", "export"] + _message_commands - ignore_line = re.compile( - r'(^$)|(^\s*#.*)|(^\s*\w+=.*)' + - r'|(^\s*(' + "|".join(_ignored_commands) + r')\s+)') - ignore_comment = False - var_names = [ - "D", "DISTDIR", "FILESDIR", "S", "T", "ROOT", "BROOT", "WORKDIR"] - - # EAPI=3/Prefix vars - var_names += ["ED", "EPREFIX", "EROOT"] - - # variables for games.eclass - var_names += [ - "Ddir", "GAMES_PREFIX_OPT", "GAMES_DATADIR", - "GAMES_DATADIR_BASE", "GAMES_SYSCONFDIR", "GAMES_STATEDIR", - "GAMES_LOGDIR", "GAMES_BINDIR"] - - # variables for multibuild.eclass - var_names += ["BUILD_DIR"] - - var_names = "(%s)" % "|".join(var_names) - var_reference = re.compile( - r'\$(\{%s\}|%s\W)' % (var_names, var_names)) - missing_quotes = re.compile( - r'(\s|^)[^"\'\s]*\$\{?%s\}?[^"\'\s]*(\s|$)' % var_names) - cond_begin = re.compile(r'(^|\s+)\[\[($|\\$|\s+)') - cond_end = re.compile(r'(^|\s+)\]\]($|\\$|\s+)') - - def check(self, num, line): - if self.var_reference.search(line) is None: - return - # There can be multiple matches / violations on a single line. We - # have to make sure none of the matches are violators. Once we've - # found one violator, any remaining matches on the same line can - # be ignored. - pos = 0 - while pos <= len(line) - 1: - missing_quotes = self.missing_quotes.search(line, pos) - if not missing_quotes: - break - # If the last character of the previous match is a whitespace - # character, that character may be needed for the next - # missing_quotes match, so search overlaps by 1 character. - group = missing_quotes.group() - pos = missing_quotes.end() - 1 - - # Filter out some false positives that can - # get through the missing_quotes regex. - if self.var_reference.search(group) is None: - continue - - # Filter matches that appear to be an - # argument to a message command. - # For example: false || ewarn "foo $WORKDIR/bar baz" - message_match = self._message_re.search(line) - if message_match is not None and \ - message_match.start() < pos and \ - message_match.end() > pos: - break - - # This is an attempt to avoid false positives without getting - # too complex, while possibly allowing some (hopefully - # unlikely) violations to slip through. We just assume - # everything is correct if the there is a ' [[ ' or a ' ]] ' - # anywhere in the whole line (possibly continued over one - # line). - if self.cond_begin.search(line) is not None: - continue - if self.cond_end.search(line) is not None: - continue - - # Any remaining matches on the same line can be ignored. - return self.errors['MISSING_QUOTES_ERROR'] diff --git a/repoman/pym/repoman/modules/linechecks/uri/__init__.py b/repoman/pym/repoman/modules/linechecks/uri/__init__.py deleted file mode 100644 index e19617257..000000000 --- a/repoman/pym/repoman/modules/linechecks/uri/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Uri plug-in module for repoman LineChecks. -Performs HOMEPAGE variable checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'do', - 'description': doc, - 'provides':{ - 'httpsuri-check': { - 'name': "httpsuri", - 'sourcefile': "uri", - 'class': "UriUseHttps", - 'description': doc, - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/linechecks/uri/uri.py b/repoman/pym/repoman/modules/linechecks/uri/uri.py deleted file mode 100644 index 1a0afe682..000000000 --- a/repoman/pym/repoman/modules/linechecks/uri/uri.py +++ /dev/null @@ -1,30 +0,0 @@ - -import re - -from repoman.modules.linechecks.base import LineCheck - - -class UriUseHttps(LineCheck): - """Check that we use https:// for known good sites.""" - repoman_check_name = 'uri.https' - _SITES = ( - '([-._a-zA-Z0-9]*\.)?apache\.org', - '((alioth|packages(\.qa)?|people|www)\.)?debian\.org', - # Most FDO sites support https, but not all (like tango). - # List the most common ones here for now. - '((anongit|bugs|cgit|dri|patchwork|people|specifications|www|xcb|xorg)\.)?freedesktop\.org', - '((bugs|dev|wiki|www)\.)?gentoo\.org', - '((wiki)\.)?github\.(io|com)', - 'savannah\.(non)?gnu\.org', - '((gcc|www)\.)?gnu\.org', - 'curl\.haxx\.se', - '((bugzilla|git|mirrors|patchwork|planet|www(\.wiki)?)\.)?kernel\.org', - '((bugs|wiki|www)\.)?linuxfoundation\.org', - '((docs|pypi|www)\.)?python\.org', - '(sf|sourceforge)\.net', - '(www\.)?(enlightenment|sourceware|x)\.org', - ) - # Try to anchor the end of the URL so we don't get false positives - # with http://github.com.foo.bar.com/. Unlikely, but possible. - re = re.compile(r'.*\bhttp://(%s)(\s|["\'/]|$)' % r'|'.join(_SITES)) - error = 'URI_HTTPS' diff --git a/repoman/pym/repoman/modules/linechecks/use/__init__.py b/repoman/pym/repoman/modules/linechecks/use/__init__.py deleted file mode 100644 index e97b0d61d..000000000 --- a/repoman/pym/repoman/modules/linechecks/use/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Use plug-in module for repoman LineChecks. -Performs Built-With-Use checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'do', - 'description': doc, - 'provides':{ - 'builtwith-check': { - 'name': "builtwith", - 'sourcefile': "builtwith", - 'class': "BuiltWithUse", - 'description': doc, - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/linechecks/use/builtwith.py b/repoman/pym/repoman/modules/linechecks/use/builtwith.py deleted file mode 100644 index 0ec4fae21..000000000 --- a/repoman/pym/repoman/modules/linechecks/use/builtwith.py +++ /dev/null @@ -1,10 +0,0 @@ - -import re - -from repoman.modules.linechecks.base import LineCheck - - -class BuiltWithUse(LineCheck): - repoman_check_name = 'ebuild.minorsyn' - re = re.compile(r'(^|.*\b)built_with_use\b') - error = 'BUILT_WITH_USE' diff --git a/repoman/pym/repoman/modules/linechecks/useless/__init__.py b/repoman/pym/repoman/modules/linechecks/useless/__init__.py deleted file mode 100644 index 5cf978137..000000000 --- a/repoman/pym/repoman/modules/linechecks/useless/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Useless plug-in module for repoman LineChecks. -Performs checks for useless operations on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'do', - 'description': doc, - 'provides':{ - 'uselesscds-check': { - 'name': "uselesscds", - 'sourcefile': "cd", - 'class': "EbuildUselessCdS", - 'description': doc, - }, - 'uselessdodoc-check': { - 'name': "uselessdodoc", - 'sourcefile': "dodoc", - 'class': "EbuildUselessDodoc", - 'description': doc, - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/linechecks/useless/cd.py b/repoman/pym/repoman/modules/linechecks/useless/cd.py deleted file mode 100644 index 3daa04451..000000000 --- a/repoman/pym/repoman/modules/linechecks/useless/cd.py +++ /dev/null @@ -1,24 +0,0 @@ - -import re - -from repoman.modules.linechecks.base import LineCheck - - -class EbuildUselessCdS(LineCheck): - """Check for redundant cd ${S} statements""" - repoman_check_name = 'ebuild.minorsyn' - _src_phases = r'^\s*src_(prepare|configure|compile|install|test)\s*\(\)' - method_re = re.compile(_src_phases) - cds_re = re.compile(r'^\s*cd\s+("\$(\{S\}|S)"|\$(\{S\}|S))\s') - - def __init__(self, errors): - self.errors = errors - self.check_next_line = False - - def check(self, num, line): - if self.check_next_line: - self.check_next_line = False - if self.cds_re.match(line): - return self.errors['REDUNDANT_CD_S_ERROR'] - elif self.method_re.match(line): - self.check_next_line = True diff --git a/repoman/pym/repoman/modules/linechecks/useless/dodoc.py b/repoman/pym/repoman/modules/linechecks/useless/dodoc.py deleted file mode 100644 index 502bfbea8..000000000 --- a/repoman/pym/repoman/modules/linechecks/useless/dodoc.py +++ /dev/null @@ -1,16 +0,0 @@ - -import re - -from repoman.modules.linechecks.base import LineCheck - - -class EbuildUselessDodoc(LineCheck): - """Check ebuild for useless files in dodoc arguments.""" - repoman_check_name = 'ebuild.minorsyn' - uselessdodoc_re = re.compile( - r'^\s*dodoc(\s+|\s+.*\s+)(ABOUT-NLS|COPYING|LICENCE|LICENSE)($|\s)') - - def check(self, num, line): - match = self.uselessdodoc_re.match(line) - if match: - return "Useless dodoc '%s'" % (match.group(2), ) + " on line: %d" diff --git a/repoman/pym/repoman/modules/linechecks/whitespace/__init__.py b/repoman/pym/repoman/modules/linechecks/whitespace/__init__.py deleted file mode 100644 index 716c5129c..000000000 --- a/repoman/pym/repoman/modules/linechecks/whitespace/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Whitespace plug-in module for repoman LineChecks. -Performs checks for useless whitespace in ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'do', - 'description': doc, - 'provides':{ - 'whitespace-check': { - 'name': "whitespace", - 'sourcefile': "whitespace", - 'class': "EbuildWhitespace", - 'description': doc, - }, - 'blankline-check': { - 'name': "blankline", - 'sourcefile': "blank", - 'class': "EbuildBlankLine", - 'description': doc, - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/linechecks/whitespace/blank.py b/repoman/pym/repoman/modules/linechecks/whitespace/blank.py deleted file mode 100644 index 2ab4097a3..000000000 --- a/repoman/pym/repoman/modules/linechecks/whitespace/blank.py +++ /dev/null @@ -1,25 +0,0 @@ - -import re - -from repoman.modules.linechecks.base import LineCheck - - -class EbuildBlankLine(LineCheck): - repoman_check_name = 'ebuild.minorsyn' - ignore_comment = False - blank_line = re.compile(r'^$') - - def new(self, pkg): - self.line_is_blank = False - - def check(self, num, line): - if self.line_is_blank and self.blank_line.match(line): - return 'Useless blank line on line: %d' - if self.blank_line.match(line): - self.line_is_blank = True - else: - self.line_is_blank = False - - def end(self): - if self.line_is_blank: - yield 'Useless blank line on last line' diff --git a/repoman/pym/repoman/modules/linechecks/whitespace/whitespace.py b/repoman/pym/repoman/modules/linechecks/whitespace/whitespace.py deleted file mode 100644 index 556b2ab81..000000000 --- a/repoman/pym/repoman/modules/linechecks/whitespace/whitespace.py +++ /dev/null @@ -1,21 +0,0 @@ - -import re - -from repoman.modules.linechecks.base import LineCheck - - -class EbuildWhitespace(LineCheck): - """Ensure ebuilds have proper whitespacing""" - - repoman_check_name = 'ebuild.minorsyn' - - ignore_line = re.compile(r'(^$)|(^(\t)*#)') - ignore_comment = False - leading_spaces = re.compile(r'^[\S\t]') - trailing_whitespace = re.compile(r'.*([\S]$)') - - def check(self, num, line): - if self.leading_spaces.match(line) is None: - return self.errors['LEADING_SPACES_ERROR'] - if self.trailing_whitespace.match(line) is None: - return self.errors['TRAILING_WHITESPACE_ERROR'] diff --git a/repoman/pym/repoman/modules/linechecks/workaround/__init__.py b/repoman/pym/repoman/modules/linechecks/workaround/__init__.py deleted file mode 100644 index 425085a5c..000000000 --- a/repoman/pym/repoman/modules/linechecks/workaround/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Workaround plug-in module for repoman LineChecks. -Performs checks for upstream workarounds in ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'do', - 'description': doc, - 'provides':{ - 'addpredict-check': { - 'name': "addpredict", - 'sourcefile': "workarounds", - 'class': "SandboxAddpredict", - 'description': doc, - }, - 'noasneeded-check': { - 'name': "noasneeded", - 'sourcefile': "workarounds", - 'class': "NoAsNeeded", - 'description': doc, - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/linechecks/workaround/workarounds.py b/repoman/pym/repoman/modules/linechecks/workaround/workarounds.py deleted file mode 100644 index 37cb54314..000000000 --- a/repoman/pym/repoman/modules/linechecks/workaround/workarounds.py +++ /dev/null @@ -1,18 +0,0 @@ - -import re - -from repoman.modules.linechecks.base import LineCheck - - -class NoAsNeeded(LineCheck): - """Check for calls to the no-as-needed function.""" - repoman_check_name = 'upstream.workaround' - re = re.compile(r'.*\$\(no-as-needed\)') - error = 'NO_AS_NEEDED' - - -class SandboxAddpredict(LineCheck): - """Check for calls to the addpredict function.""" - repoman_check_name = 'upstream.workaround' - re = re.compile(r'(^|\s)addpredict\b') - error = 'SANDBOX_ADDPREDICT' diff --git a/repoman/pym/repoman/modules/scan/__init__.py b/repoman/pym/repoman/modules/scan/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/repoman/pym/repoman/modules/scan/__init__.py +++ /dev/null diff --git a/repoman/pym/repoman/modules/scan/depend/__init__.py b/repoman/pym/repoman/modules/scan/depend/__init__.py deleted file mode 100644 index c3cc0ddeb..000000000 --- a/repoman/pym/repoman/modules/scan/depend/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Depend plug-in module for repoman. -Performs Dependency checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'depend', - 'description': doc, - 'provides':{ - 'profile-module': { - 'name': "profile", - 'sourcefile': "profile", - 'class': "ProfileDependsChecks", - 'description': doc, - 'functions': ['check'], - 'func_desc': { - }, - 'mod_kwargs': ['qatracker', 'portdb', 'profiles', 'options', - 'repo_metadata', 'repo_settings', 'include_arches', 'caches', - 'repoman_incrementals', 'env', 'have', 'dev_keywords' - ], - 'func_kwargs': { - 'ebuild': (None, None), - 'pkg': (None, None), - }, - 'module_runsIn': ['ebuilds'], - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/scan/depend/_depend_checks.py b/repoman/pym/repoman/modules/scan/depend/_depend_checks.py deleted file mode 100644 index 690b95aa0..000000000 --- a/repoman/pym/repoman/modules/scan/depend/_depend_checks.py +++ /dev/null @@ -1,202 +0,0 @@ -# -*- coding:utf-8 -*- - -import collections - -from _emerge.Package import Package - -from portage.dep import Atom - -from repoman.check_missingslot import check_missingslot -# import our initialized portage instance -from repoman._portage import portage - -def check_slotop(depstr, is_valid_flag, badsyntax, mytype, - qatracker, relative_path): - '''Checks if RDEPEND uses ':=' slot operator - in '||' style dependencies.''' - - try: - # to find use of ':=' in '||' we preserve - # tree structure of dependencies - my_dep_tree = portage.dep.use_reduce( - depstr, - flat=False, - matchall=1, - is_valid_flag=is_valid_flag, - opconvert=True, - token_class=portage.dep.Atom) - except portage.exception.InvalidDependString as e: - my_dep_tree = None - badsyntax.append((mytype, str(e))) - - def _traverse_tree(dep_tree, in_any_of): - # leaf - if isinstance(dep_tree, Atom): - atom = dep_tree - if in_any_of and atom.slot_operator == '=': - qatracker.add_error("dependency.badslotop", - "%s: %s: '%s' uses ':=' slot operator under '||' dep clause." % - (relative_path, mytype, atom)) - - # branches - if isinstance(dep_tree, list): - if len(dep_tree) == 0: - return - # entering any-of - if dep_tree[0] == '||': - _traverse_tree(dep_tree[1:], in_any_of=True) - else: - for branch in dep_tree: - _traverse_tree(branch, in_any_of=in_any_of) - _traverse_tree(my_dep_tree, False) - -def _depend_checks(ebuild, pkg, portdb, qatracker, repo_metadata, qadata): - '''Checks the ebuild dependencies for errors - - @param pkg: Package in which we check (object). - @param ebuild: Ebuild which we check (object). - @param portdb: portdb instance - @param qatracker: QATracker instance - @param repo_metadata: dictionary of various repository items. - @returns: (unknown_pkgs, badlicsyntax) - ''' - - unknown_pkgs = set() - - inherited_java_eclass = "java-pkg-2" in ebuild.inherited or \ - "java-pkg-opt-2" in ebuild.inherited, - inherited_wxwidgets_eclass = "wxwidgets" in ebuild.inherited - # operator_tokens = set(["||", "(", ")"]) - badsyntax = [] - for mytype in Package._dep_keys + ("LICENSE", "PROPERTIES"): - mydepstr = ebuild.metadata[mytype] - - buildtime = mytype in Package._buildtime_keys - runtime = mytype in Package._runtime_keys - token_class = None - if mytype.endswith("DEPEND"): - token_class = portage.dep.Atom - - try: - atoms = portage.dep.use_reduce( - mydepstr, matchall=1, flat=True, - is_valid_flag=pkg.iuse.is_valid_flag, token_class=token_class) - except portage.exception.InvalidDependString as e: - atoms = None - badsyntax.append((mytype, str(e))) - - if atoms and mytype.endswith("DEPEND"): - if runtime and \ - "test?" in mydepstr.split(): - qatracker.add_error( - mytype + '.suspect', - "%s: 'test?' USE conditional in %s" % - (ebuild.relative_path, mytype)) - - for atom in atoms: - if atom == "||": - continue - - is_blocker = atom.blocker - - # Skip dependency.unknown for blockers, so that we - # don't encourage people to remove necessary blockers, - # as discussed in bug 382407. We use atom.without_use - # due to bug 525376. - if not is_blocker and \ - not portdb.xmatch("match-all", atom.without_use) and \ - not atom.cp.startswith("virtual/"): - unknown_pkgs.add((mytype, atom.unevaluated_atom)) - - if pkg.category != "virtual": - if not is_blocker and \ - atom.cp in qadata.suspect_virtual: - qatracker.add_error( - 'virtual.suspect', ebuild.relative_path + - ": %s: consider using '%s' instead of '%s'" % - (mytype, qadata.suspect_virtual[atom.cp], atom)) - if not is_blocker and \ - atom.cp.startswith("perl-core/"): - qatracker.add_error('dependency.perlcore', - ebuild.relative_path + - ": %s: please use '%s' instead of '%s'" % - (mytype, - atom.replace("perl-core/","virtual/perl-"), - atom)) - - if buildtime and \ - not is_blocker and \ - not inherited_java_eclass and \ - atom.cp == "virtual/jdk": - qatracker.add_error( - 'java.eclassesnotused', ebuild.relative_path) - elif buildtime and \ - not is_blocker and \ - not inherited_wxwidgets_eclass and \ - atom.cp == "x11-libs/wxGTK": - qatracker.add_error( - 'wxwidgets.eclassnotused', - "%s: %ss on x11-libs/wxGTK without inheriting" - " wxwidgets.eclass" % (ebuild.relative_path, mytype)) - elif runtime: - if not is_blocker and \ - atom.cp in qadata.suspect_rdepend: - qatracker.add_error( - mytype + '.suspect', - ebuild.relative_path + ": '%s'" % atom) - - if atom.operator == "~" and \ - portage.versions.catpkgsplit(atom.cpv)[3] != "r0": - qacat = 'dependency.badtilde' - qatracker.add_error( - qacat, "%s: %s uses the ~ operator" - " with a non-zero revision: '%s'" % - (ebuild.relative_path, mytype, atom)) - # plain =foo-1.2.3 without revision or * - if atom.operator == "=" and '-r' not in atom.version: - qacat = 'dependency.equalsversion' - qatracker.add_error( - qacat, "%s: %s uses the = operator with" - " no revision: '%s'; if any revision is" - " acceptable, use '~' instead; if only -r0" - " then please append '-r0' to the dep" % - (ebuild.relative_path, mytype, atom)) - - check_missingslot(atom, mytype, ebuild.eapi, portdb, qatracker, - ebuild.relative_path, ebuild.metadata) - - if runtime: - check_slotop(mydepstr, pkg.iuse.is_valid_flag, - badsyntax, mytype, qatracker, ebuild.relative_path) - - baddepsyntax = False - dedup = collections.defaultdict(set) - for m, b in badsyntax: - if b in dedup[m]: - continue - dedup[m].add(b) - - if m.endswith("DEPEND"): - baddepsyntax = True - qacat = "dependency.syntax" - else: - qacat = m + ".syntax" - qatracker.add_error( - qacat, "%s: %s: %s" % (ebuild.relative_path, m, b)) - - # Parse the LICENSE variable, remove USE conditions and flatten it. - licenses = portage.dep.use_reduce( - ebuild.metadata["LICENSE"], matchall=1, flat=True) - - # Check each entry to ensure that it exists in ${PORTDIR}/licenses/. - for lic in licenses: - # Need to check for "||" manually as no portage - # function will remove it without removing values. - if lic not in repo_metadata['liclist'] and lic != "||": - qatracker.add_error("LICENSE.invalid", - "%s: %s" % (ebuild.relative_path, lic)) - elif lic in repo_metadata['lic_deprecated']: - qatracker.add_error("LICENSE.deprecated", - "%s: %s" % (ebuild.relative_path, lic)) - - return unknown_pkgs, baddepsyntax diff --git a/repoman/pym/repoman/modules/scan/depend/_gen_arches.py b/repoman/pym/repoman/modules/scan/depend/_gen_arches.py deleted file mode 100644 index 16b8dac5f..000000000 --- a/repoman/pym/repoman/modules/scan/depend/_gen_arches.py +++ /dev/null @@ -1,57 +0,0 @@ -# -*- coding:utf-8 -*- - - -def _gen_arches(ebuild, options, repo_settings, profiles): - '''Determines the arches for the ebuild following the profile rules - - @param ebuild: Ebuild which we check (object). - @param profiles: dictionary - @param options: cli options - @param repo_settings: repository settings instance - @returns: dictionary, including arches set - ''' - if options.ignore_arches: - arches = [[ - repo_settings.repoman_settings["ARCH"], repo_settings.repoman_settings["ARCH"], - repo_settings.repoman_settings["ACCEPT_KEYWORDS"].split()]] - else: - arches = set() - for keyword in ebuild.keywords: - if keyword[0] == "-": - continue - elif keyword[0] == "~": - arch = keyword[1:] - if arch == "*": - for expanded_arch in profiles: - if expanded_arch == "**": - continue - arches.add( - (keyword, expanded_arch, ( - expanded_arch, "~" + expanded_arch))) - else: - arches.add((keyword, arch, (arch, keyword))) - else: - # For ebuilds with stable keywords, check if the - # dependencies are satisfiable for unstable - # configurations, since use.stable.mask is not - # applied for unstable configurations (see bug - # 563546). - if keyword == "*": - for expanded_arch in profiles: - if expanded_arch == "**": - continue - arches.add( - (keyword, expanded_arch, (expanded_arch,))) - arches.add( - (keyword, expanded_arch, - (expanded_arch, "~" + expanded_arch))) - else: - arches.add((keyword, keyword, (keyword,))) - arches.add((keyword, keyword, - (keyword, "~" + keyword))) - if not arches: - # Use an empty profile for checking dependencies of - # packages that have empty KEYWORDS. - arches.add(('**', '**', ('**',))) - - return arches diff --git a/repoman/pym/repoman/modules/scan/depend/profile.py b/repoman/pym/repoman/modules/scan/depend/profile.py deleted file mode 100644 index d980f4eca..000000000 --- a/repoman/pym/repoman/modules/scan/depend/profile.py +++ /dev/null @@ -1,295 +0,0 @@ -# -*- coding:utf-8 -*- - - -import copy -import os -from pprint import pformat - -from _emerge.Package import Package - -# import our initialized portage instance -from repoman._portage import portage -from repoman.modules.scan.scanbase import ScanBase -from repoman.modules.scan.depend._depend_checks import _depend_checks -from repoman.modules.scan.depend._gen_arches import _gen_arches -from portage.dep import Atom -from portage.package.ebuild.profile_iuse import iter_iuse_vars -from portage.util import getconfig - - -def sort_key(item): - return item[2].sub_path - - -class ProfileDependsChecks(ScanBase): - '''Perform dependency checks for the different profiles''' - - def __init__(self, **kwargs): - '''Class init - - @param qatracker: QATracker instance - @param portdb: portdb instance - @param profiles: dictionary - @param options: cli options - @param repo_settings: repository settings instance - @param include_arches: set - @param caches: dictionary of our caches - @param repoman_incrementals: tuple - @param env: the environment - @param have: dictionary instance - @param dev_keywords: developer profile keywords - @param repo_metadata: dictionary of various repository items. - ''' - self.qatracker = kwargs.get('qatracker') - self.portdb = kwargs.get('portdb') - self.profiles = kwargs.get('profiles') - self.options = kwargs.get('options') - self.repo_settings = kwargs.get('repo_settings') - self.include_arches = kwargs.get('include_arches') - self.caches = kwargs.get('caches') - self.repoman_incrementals = kwargs.get('repoman_incrementals') - self.env = kwargs.get('env') - self.have = kwargs.get('have') - self.dev_keywords = kwargs.get('dev_keywords') - self.repo_metadata = kwargs.get('repo_metadata') - - def check(self, **kwargs): - '''Perform profile dependant dependency checks - - @param arches: - @param pkg: Package in which we check (object). - @param ebuild: Ebuild which we check (object). - @param baddepsyntax: boolean - @param unknown_pkgs: set of tuples (type, atom.unevaluated_atom) - @returns: dictionary - ''' - ebuild = kwargs.get('ebuild').get() - pkg = kwargs.get('pkg').get() - unknown_pkgs, baddepsyntax = _depend_checks( - ebuild, pkg, self.portdb, self.qatracker, self.repo_metadata, - self.repo_settings.qadata) - - relevant_profiles = [] - for keyword, arch, groups in _gen_arches(ebuild, self.options, - self.repo_settings, self.profiles): - if arch not in self.profiles: - # A missing profile will create an error further down - # during the KEYWORDS verification. - continue - - if self.include_arches is not None: - if arch not in self.include_arches: - continue - - relevant_profiles.extend( - (keyword, groups, prof) for prof in self.profiles[arch]) - - relevant_profiles.sort(key=sort_key) - - for keyword, groups, prof in relevant_profiles: - - is_stable_profile = prof.status == "stable" - is_dev_profile = prof.status == "dev" and \ - self.options.include_dev - is_exp_profile = prof.status == "exp" and \ - self.options.include_exp_profiles == 'y' - if not (is_stable_profile or is_dev_profile or is_exp_profile): - continue - - dep_settings = self.caches['arch'].get(prof.sub_path) - if dep_settings is None: - dep_settings = portage.config( - config_profile_path=prof.abs_path, - config_incrementals=self.repoman_incrementals, - config_root=self.repo_settings.config_root, - local_config=False, - _unmatched_removal=self.options.unmatched_removal, - env=self.env, repositories=self.repo_settings.repoman_settings.repositories) - - if not prof.abs_path: - self._populate_implicit_iuse(dep_settings, - self.repo_settings.repo_config.eclass_db.porttrees) - - dep_settings.categories = self.repo_settings.repoman_settings.categories - if self.options.without_mask: - dep_settings._mask_manager_obj = \ - copy.deepcopy(dep_settings._mask_manager) - dep_settings._mask_manager._pmaskdict.clear() - self.caches['arch'][prof.sub_path] = dep_settings - - xmatch_cache_key = (prof.sub_path, tuple(groups)) - xcache = self.caches['arch_xmatch'].get(xmatch_cache_key) - if xcache is None: - self.portdb.melt() - self.portdb.freeze() - xcache = self.portdb.xcache - xcache.update(self.caches['shared_xmatch']) - self.caches['arch_xmatch'][xmatch_cache_key] = xcache - - self.repo_settings.trees[self.repo_settings.root]["porttree"].settings = dep_settings - self.portdb.settings = dep_settings - self.portdb.xcache = xcache - - dep_settings["ACCEPT_KEYWORDS"] = " ".join(groups) - # just in case, prevent config.reset() from nuking these. - dep_settings.backup_changes("ACCEPT_KEYWORDS") - - # This attribute is used in dbapi._match_use() to apply - # use.stable.{mask,force} settings based on the stable - # status of the parent package. This is required in order - # for USE deps of unstable packages to be resolved correctly, - # since otherwise use.stable.{mask,force} settings of - # dependencies may conflict (see bug #456342). - dep_settings._parent_stable = dep_settings._isStable(pkg) - - # Handle package.use*.{force,mask) calculation, for use - # in dep_check. - dep_settings.useforce = dep_settings._use_manager.getUseForce( - pkg, stable=dep_settings._parent_stable) - dep_settings.usemask = dep_settings._use_manager.getUseMask( - pkg, stable=dep_settings._parent_stable) - - if not baddepsyntax: - ismasked = not ebuild.archs or \ - pkg.cpv not in self.portdb.xmatch("match-visible", - Atom("%s::%s" % (pkg.cp, self.repo_settings.repo_config.name))) - if ismasked: - if not self.have['pmasked']: - self.have['pmasked'] = bool(dep_settings._getMaskAtom( - pkg.cpv, ebuild.metadata)) - if self.options.ignore_masked: - continue - # we are testing deps for a masked package; give it some lee-way - suffix = "masked" - matchmode = "minimum-all-ignore-profile" - else: - suffix = "" - matchmode = "minimum-visible" - - if not self.have['dev_keywords']: - self.have['dev_keywords'] = \ - bool(self.dev_keywords.intersection(ebuild.keywords)) - - if prof.status == "dev": - suffix = suffix + "indev" - elif prof.status == "exp": - suffix = suffix + "inexp" - - for mytype in Package._dep_keys: - - mykey = "dependency.bad" + suffix - myvalue = ebuild.metadata[mytype] - if not myvalue: - continue - - success, atoms = portage.dep_check( - myvalue, self.portdb, dep_settings, - use="all", mode=matchmode, trees=self.repo_settings.trees) - - if success: - if atoms: - - # Don't bother with dependency.unknown for - # cases in which *DEPEND.bad is triggered. - for atom in atoms: - # dep_check returns all blockers and they - # aren't counted for *DEPEND.bad, so we - # ignore them here. - if not atom.blocker: - unknown_pkgs.discard( - (mytype, atom.unevaluated_atom)) - - if not prof.sub_path: - # old-style virtuals currently aren't - # resolvable with empty profile, since - # 'virtuals' mappings are unavailable - # (it would be expensive to search - # for PROVIDE in all ebuilds) - atoms = [ - atom for atom in atoms if not ( - atom.cp.startswith('virtual/') - and not self.portdb.cp_list(atom.cp))] - - # we have some unsolvable deps - # remove ! deps, which always show up as unsatisfiable - all_atoms = [ - str(atom.unevaluated_atom) - for atom in atoms if not atom.blocker] - - # if we emptied out our list, continue: - if not all_atoms: - continue - - # Filter out duplicates. We do this by hand (rather - # than use a set) so the order is stable and better - # matches the order that's in the ebuild itself. - atoms = [] - for atom in all_atoms: - if atom not in atoms: - atoms.append(atom) - - if self.options.output_style in ['column']: - self.qatracker.add_error(mykey, - "%s: %s: %s(%s) %s" - % (ebuild.relative_path, mytype, keyword, - prof, repr(atoms))) - else: - self.qatracker.add_error(mykey, - "%s: %s: %s(%s)\n%s" - % (ebuild.relative_path, mytype, keyword, - prof, pformat(atoms, indent=6))) - else: - if self.options.output_style in ['column']: - self.qatracker.add_error(mykey, - "%s: %s: %s(%s) %s" - % (ebuild.relative_path, mytype, keyword, - prof, repr(atoms))) - else: - self.qatracker.add_error(mykey, - "%s: %s: %s(%s)\n%s" - % (ebuild.relative_path, mytype, keyword, - prof, pformat(atoms, indent=6))) - - if not baddepsyntax and unknown_pkgs: - type_map = {} - for mytype, atom in unknown_pkgs: - type_map.setdefault(mytype, set()).add(atom) - for mytype, atoms in type_map.items(): - self.qatracker.add_error( - "dependency.unknown", "%s: %s: %s" - % (ebuild.relative_path, mytype, ", ".join(sorted(atoms)))) - - return False - - @property - def runInEbuilds(self): - '''Ebuild level scans''' - return (True, [self.check]) - - @staticmethod - def _populate_implicit_iuse(config, repo_locations): - """ - Populate implicit IUSE for the empty profile, see bug 660982. - - @param config: config instance for the empty profile - @type config: portage.config - @param repo_locations: locations of repositories containing relevant - implicit IUSE settings - @type repo_locations: list - """ - dest = config.configdict['defaults'] - for location in repo_locations: - for parent_dir, dirs, files in os.walk(os.path.join(location, 'profiles')): - src = getconfig(os.path.join(parent_dir, 'make.defaults')) - if not src: - continue - for k, v in iter_iuse_vars(src): - v_before = dest.get(k) - if v_before is not None: - merged_values = set(v_before.split()) - merged_values.update(v.split()) - v = ' '.join(sorted(merged_values)) - dest[k] = v - - config.regenerate() - config._init_iuse() diff --git a/repoman/pym/repoman/modules/scan/directories/__init__.py b/repoman/pym/repoman/modules/scan/directories/__init__.py deleted file mode 100644 index 276bc7ae9..000000000 --- a/repoman/pym/repoman/modules/scan/directories/__init__.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Directories plug-in module for repoman. -Performs an FilesChecks check on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'directories', - 'description': doc, - 'provides':{ - 'directories-module': { - 'name': "files", - 'sourcefile': "files", - 'class': "FileChecks", - 'description': doc, - 'functions': ['check'], - 'func_kwargs': { - }, - 'mod_kwargs': ['portdb', 'qatracker', 'repo_settings', 'vcs_settings', - ], - 'func_kwargs': { - 'changed': (None, None), - 'checkdir': (None, None), - 'checkdirlist': (None, None), - 'checkdir_relative': (None, None), - }, - 'module_runsIn': ['pkgs'], - }, - 'mtime-module': { - 'name': "mtime", - 'sourcefile': "mtime", - 'class': "MtimeChecks", - 'description': doc, - 'functions': ['check'], - 'func_kwargs': { - }, - 'mod_kwargs': ['vcs_settings', - ], - 'func_kwargs': { - 'changed': (None, None), - 'ebuild': (None, None), - 'pkg': (None, None), - }, - 'module_runsIn': ['ebuilds'], - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/scan/directories/files.py b/repoman/pym/repoman/modules/scan/directories/files.py deleted file mode 100644 index 2aed26440..000000000 --- a/repoman/pym/repoman/modules/scan/directories/files.py +++ /dev/null @@ -1,94 +0,0 @@ -# -*- coding:utf-8 -*- - -'''repoman/checks/diretories/files.py - -''' - -import io - -from portage import _encodings, _unicode_encode -from portage import os - -from repoman.modules.vcs.vcs import vcs_new_changed -from repoman.modules.scan.scanbase import ScanBase - - -class FileChecks(ScanBase): - '''Performs various file checks in the package's directory''' - - def __init__(self, **kwargs): - ''' - @param portdb: portdb instance - @param qatracker: QATracker instance - @param repo_settings: settings instance - @param vcs_settings: VCSSettings instance - ''' - super(FileChecks, self).__init__(**kwargs) - self.portdb = kwargs.get('portdb') - self.qatracker = kwargs.get('qatracker') - self.repo_settings = kwargs.get('repo_settings') - self.repoman_settings = self.repo_settings.repoman_settings - self.vcs_settings = kwargs.get('vcs_settings') - - def check(self, **kwargs): - '''Checks the ebuild sources and files for errors - - @param checkdir: string, directory path - @param checkdir_relative: repolevel determined path - @param changed: dictionary instance - @returns: dictionary - ''' - checkdir = kwargs.get('checkdir') - checkdirlist = kwargs.get('checkdirlist').get() - checkdir_relative = kwargs.get('checkdir_relative') - changed = kwargs.get('changed').changed - new = kwargs.get('changed').new - for y_file in checkdirlist: - index = self.repo_settings.repo_config.find_invalid_path_char(y_file) - if index != -1: - y_relative = os.path.join(checkdir_relative, y_file) - invcs = self.vcs_settings.vcs is not None - inchangeset = vcs_new_changed(y_relative, changed, new) - if invcs and not inchangeset: - # If the file isn't in the VCS new or changed set, then - # assume that it's an irrelevant temporary file (Manifest - # entries are not generated for file names containing - # prohibited characters). See bug #406877. - index = -1 - if index != -1: - self.qatracker.add_error( - "file.name", - "%s/%s: char '%s'" % (checkdir, y_file, y_file[index])) - - if not ( - y_file in ("ChangeLog", "metadata.xml") - or y_file.endswith(".ebuild")): - continue - f = None - try: - line = 1 - f = io.open( - _unicode_encode( - os.path.join(checkdir, y_file), - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content']) - for l in f: - line += 1 - except UnicodeDecodeError as ue: - s = ue.object[:ue.start] - l2 = s.count("\n") - line += l2 - if l2 != 0: - s = s[s.rfind("\n") + 1:] - self.qatracker.add_error( - "file.UTF8", "%s/%s: line %i, just after: '%s'" % ( - checkdir, y_file, line, s)) - finally: - if f is not None: - f.close() - return False - - @property - def runInPkgs(self): - '''Package level scans''' - return (True, [self.check]) diff --git a/repoman/pym/repoman/modules/scan/directories/mtime.py b/repoman/pym/repoman/modules/scan/directories/mtime.py deleted file mode 100644 index 134a86b80..000000000 --- a/repoman/pym/repoman/modules/scan/directories/mtime.py +++ /dev/null @@ -1,30 +0,0 @@ - -from repoman.modules.scan.scanbase import ScanBase - - -class MtimeChecks(ScanBase): - - def __init__(self, **kwargs): - self.vcs_settings = kwargs.get('vcs_settings') - - def check(self, **kwargs): - '''Perform a changelog and untracked checks on the ebuild - - @param pkg: Package in which we check (object). - @param ebuild: Ebuild which we check (object). - @param changed: dictionary instance - @returns: dictionary - ''' - ebuild = kwargs.get('ebuild').get() - changed = kwargs.get('changed') - pkg = kwargs.get('pkg').get() - if not self.vcs_settings.vcs_preserves_mtime: - if ebuild.ebuild_path not in changed.new_ebuilds and \ - ebuild.ebuild_path not in changed.ebuilds: - pkg.mtime = None - return False - - @property - def runInEbuilds(self): - '''Ebuild level scans''' - return (True, [self.check]) diff --git a/repoman/pym/repoman/modules/scan/eapi/__init__.py b/repoman/pym/repoman/modules/scan/eapi/__init__.py deleted file mode 100644 index 94759fdd7..000000000 --- a/repoman/pym/repoman/modules/scan/eapi/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Eapi plug-in module for repoman. -Performs an IsEbuild check on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'eapi', - 'description': doc, - 'provides':{ - 'live-module': { - 'name': "eapi", - 'sourcefile': "eapi", - 'class': "EAPIChecks", - 'description': doc, - 'functions': ['check'], - 'func_kwargs': { - }, - 'mod_kwargs': ['qatracker', 'repo_settings' - ], - 'func_kwargs': { - 'ebuild': (None, None), - }, - 'module_runsIn': ['ebuilds'], - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/scan/eapi/eapi.py b/repoman/pym/repoman/modules/scan/eapi/eapi.py deleted file mode 100644 index fbad502a9..000000000 --- a/repoman/pym/repoman/modules/scan/eapi/eapi.py +++ /dev/null @@ -1,49 +0,0 @@ - -'''eapi.py -Perform checks on the EAPI variable. -''' - -from repoman.modules.scan.scanbase import ScanBase - - -class EAPIChecks(ScanBase): - '''Perform checks on the EAPI variable.''' - - def __init__(self, **kwargs): - ''' - @param qatracker: QATracker instance - @param repo_settings: Repository settings - ''' - self.qatracker = kwargs.get('qatracker') - self.repo_settings = kwargs.get('repo_settings') - - def check(self, **kwargs): - ''' - @param pkg: Package in which we check (object). - @param ebuild: Ebuild which we check (object). - @returns: dictionary - ''' - ebuild = kwargs.get('ebuild').get() - - if not self._checkBanned(ebuild): - self._checkDeprecated(ebuild) - return False - - def _checkBanned(self, ebuild): - if self.repo_settings.repo_config.eapi_is_banned(ebuild.eapi): - self.qatracker.add_error( - "repo.eapi-banned", "%s: %s" % (ebuild.relative_path, ebuild.eapi)) - return True - return False - - def _checkDeprecated(self, ebuild): - if self.repo_settings.repo_config.eapi_is_deprecated(ebuild.eapi): - self.qatracker.add_error( - "repo.eapi-deprecated", "%s: %s" % (ebuild.relative_path, ebuild.eapi)) - return True - return False - - @property - def runInEbuilds(self): - '''Ebuild level scans''' - return (True, [self.check]) diff --git a/repoman/pym/repoman/modules/scan/ebuild/__init__.py b/repoman/pym/repoman/modules/scan/ebuild/__init__.py deleted file mode 100644 index 328a0d626..000000000 --- a/repoman/pym/repoman/modules/scan/ebuild/__init__.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Ebuild plug-in module for repoman. -Performs an IsEbuild check on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'ebuild', - 'description': doc, - 'provides':{ - 'ebuild-module': { - 'name': "ebuild", - 'sourcefile': "ebuild", - 'class': "Ebuild", - 'description': doc, - 'functions': ['check'], - 'func_desc': { - }, - 'mod_kwargs': ['qatracker', 'repo_settings', 'vcs_settings', - 'checks', 'portdb' - ], - 'func_kwargs': { - 'can_force': (None, None), - 'catdir': (None, None), - 'changed': (None, None), - 'changelog_modified': (None, None), - 'checkdir': (None, None), - 'checkdirlist': (None, None), - 'ebuild': ('Future', 'UNSET'), - 'pkg': ('Future', 'UNSET'), - 'pkgdir': (None, None), - 'pkgs': ('Future', 'dict'), - 'repolevel': (None, None), - 'validity_future': (None, None), - 'xpkg': (None, None), - 'y_ebuild': (None, None), - }, - 'module_runsIn': ['pkgs', 'ebuilds'], - }, - 'multicheck-module': { - 'name': "multicheck", - 'sourcefile': "multicheck", - 'class': "MultiCheck", - 'description': doc, - 'functions': ['check'], - 'func_kwargs': { - }, - 'mod_kwargs': ['qatracker', 'options', 'repo_settings', 'linechecks', - ], - 'func_kwargs': { - 'ebuild': (None, None), - 'pkg': (None, None), - }, - 'module_runsIn': ['ebuilds'], - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/scan/ebuild/ebuild.py b/repoman/pym/repoman/modules/scan/ebuild/ebuild.py deleted file mode 100644 index d2715bc6e..000000000 --- a/repoman/pym/repoman/modules/scan/ebuild/ebuild.py +++ /dev/null @@ -1,239 +0,0 @@ -# -*- coding:utf-8 -*- - -from __future__ import print_function, unicode_literals - -import re -import stat - -from _emerge.Package import Package -from _emerge.RootConfig import RootConfig - -from repoman.modules.scan.scanbase import ScanBase -# import our initialized portage instance -from repoman._portage import portage -from portage import os -from portage.const import LIVE_ECLASSES -from portage.exception import InvalidPackageName - -pv_toolong_re = re.compile(r'[0-9]{19,}') - - -class Ebuild(ScanBase): - '''Class to run primary checks on ebuilds''' - - def __init__(self, **kwargs): - '''Class init - - @param qatracker: QATracker instance - @param portdb: portdb instance - @param repo_settings: repository settings instance - @param vcs_settings: VCSSettings instance - @param checks: checks dictionary - ''' - super(Ebuild, self).__init__(**kwargs) - self.qatracker = kwargs.get('qatracker') - self.portdb = kwargs.get('portdb') - self.repo_settings = kwargs.get('repo_settings') - self.vcs_settings = kwargs.get('vcs_settings') - self.checks = kwargs.get('checks') - self.root_config = RootConfig(self.repo_settings.repoman_settings, - self.repo_settings.trees[self.repo_settings.root], None) - self.changed = None - self.xpkg = None - self.y_ebuild = None - self.pkg = None - self.metadata = None - self.eapi = None - self.inherited = None - self.live_ebuild = None - self.keywords = None - self.pkgs = {} - - def _set_paths(self, **kwargs): - repolevel = kwargs.get('repolevel') - self.relative_path = os.path.join(self.xpkg, self.y_ebuild + ".ebuild") - self.full_path = os.path.join(self.repo_settings.repodir, self.relative_path) - self.ebuild_path = self.y_ebuild + ".ebuild" - if repolevel < 3: - self.ebuild_path = os.path.join(kwargs.get('pkgdir'), self.ebuild_path) - if repolevel < 2: - self.ebuild_path = os.path.join(kwargs.get('catdir'), self.ebuild_path) - self.ebuild_path = os.path.join(".", self.ebuild_path) - - @property - def untracked(self): - '''Determines and returns if the ebuild is not tracked by the vcs''' - do_check = self.vcs_settings.vcs in ("cvs", "svn", "bzr") - really_notadded = (self.checks['ebuild_notadded'] and - self.y_ebuild not in self.vcs_settings.eadded) - if do_check and really_notadded: - # ebuild not added to vcs - return True - return False - - def check(self, **kwargs): - '''Perform a changelog and untracked checks on the ebuild - - @param xpkg: Package in which we check (object). - @param y_ebuild: Ebuild which we check (string). - @param changed: dictionary instance - @param repolevel: The depth within the repository - @param catdir: The category directiory - @param pkgdir: the package directory - @returns: dictionary, including {ebuild object} - ''' - self.xpkg = kwargs.get('xpkg') - self.y_ebuild = kwargs.get('y_ebuild') - self.changed = kwargs.get('changed') - changelog_modified = kwargs.get('changelog_modified') - self._set_paths(**kwargs) - - if self.checks['changelog'] and not changelog_modified \ - and self.ebuild_path in self.changed.new_ebuilds: - self.qatracker.add_error('changelog.ebuildadded', self.relative_path) - - if self.untracked: - # ebuild not added to vcs - self.qatracker.add_error( - "ebuild.notadded", self.xpkg + "/" + self.y_ebuild + ".ebuild") - # update the dynamic data - dyn_ebuild = kwargs.get('ebuild') - dyn_ebuild.set(self) - return False - - def set_pkg_data(self, **kwargs): - '''Sets some classwide data needed for some of the checks - - @returns: dictionary - ''' - self.pkg = self.pkgs[self.y_ebuild] - self.metadata = self.pkg._metadata - self.eapi = self.metadata["EAPI"] - self.inherited = self.pkg.inherited - self.live_ebuild = LIVE_ECLASSES.intersection(self.inherited) - self.keywords = self.metadata["KEYWORDS"].split() - self.archs = set(kw.lstrip("~") for kw in self.keywords if not kw.startswith("-")) - return False - - def bad_split_check(self, **kwargs): - '''Checks for bad category/package splits. - - @param pkgdir: string: path - @returns: dictionary - ''' - pkgdir = kwargs.get('pkgdir') - myesplit = portage.pkgsplit(self.y_ebuild) - is_bad_split = myesplit is None or myesplit[0] != self.xpkg.split("/")[-1] - if is_bad_split: - is_pv_toolong = pv_toolong_re.search(myesplit[1]) - is_pv_toolong2 = pv_toolong_re.search(myesplit[2]) - if is_pv_toolong or is_pv_toolong2: - self.qatracker.add_error( - "ebuild.invalidname", self.xpkg + "/" + self.y_ebuild + ".ebuild") - return True - elif myesplit[0] != pkgdir: - print(pkgdir, myesplit[0]) - self.qatracker.add_error( - "ebuild.namenomatch", self.xpkg + "/" + self.y_ebuild + ".ebuild") - return True - return False - - def pkg_invalid(self, **kwargs): - '''Sets some pkg info and checks for invalid packages - - @param validity_future: Future instance - @returns: dictionary, including {pkg object} - ''' - fuse = kwargs.get('validity_future') - dyn_pkg = kwargs.get('pkg') - if self.pkg.invalid: - for k, msgs in self.pkg.invalid.items(): - for msg in msgs: - self.qatracker.add_error(k, "%s: %s" % (self.relative_path, msg)) - # update the dynamic data - fuse.set(False, ignore_InvalidState=True) - dyn_pkg.set(self.pkg) - return True - # update the dynamic data - dyn_pkg.set(self.pkg) - return False - - def check_isebuild(self, **kwargs): - '''Test the file for qualifications that is is an ebuild - - @param checkdirlist: list of files in the current package directory - @param checkdir: current package directory path - @param xpkg: current package directory being checked - @param validity_future: Future instance - @returns: dictionary, including {pkgs, can_force} - ''' - checkdirlist = kwargs.get('checkdirlist').get() - checkdir = kwargs.get('checkdir') - xpkg = kwargs.get('xpkg') - fuse = kwargs.get('validity_future') - can_force = kwargs.get('can_force') - self.continue_ = False - ebuildlist = [] - pkgs = {} - for y in checkdirlist: - file_is_ebuild = y.endswith(".ebuild") - file_should_be_non_executable = ( - y in self.repo_settings.qadata.no_exec or file_is_ebuild) - - if file_should_be_non_executable: - file_is_executable = stat.S_IMODE( - os.stat(os.path.join(checkdir, y)).st_mode) & 0o111 - - if file_is_executable: - self.qatracker.add_error("file.executable", os.path.join(checkdir, y)) - if file_is_ebuild: - pf = y[:-7] - ebuildlist.append(pf) - catdir = xpkg.split("/")[0] - cpv = "%s/%s" % (catdir, pf) - allvars = self.repo_settings.qadata.allvars - try: - myaux = dict(zip(allvars, self.portdb.aux_get(cpv, allvars))) - except KeyError: - fuse.set(False, ignore_InvalidState=True) - self.qatracker.add_error("ebuild.syntax", os.path.join(xpkg, y)) - continue - except IOError: - fuse.set(False, ignore_InvalidState=True) - self.qatracker.add_error("ebuild.output", os.path.join(xpkg, y)) - continue - except InvalidPackageName: - fuse.set(False, ignore_InvalidState=True) - self.qatracker.add_error("ebuild.invalidname", os.path.join(xpkg, y)) - continue - if not portage.eapi_is_supported(myaux["EAPI"]): - fuse.set(False, ignore_InvalidState=True) - self.qatracker.add_error("EAPI.unsupported", os.path.join(xpkg, y)) - continue - pkgs[pf] = Package( - cpv=cpv, metadata=myaux, root_config=self.root_config, - type_name="ebuild") - - if len(pkgs) != len(ebuildlist): - # If we can't access all the metadata then it's totally unsafe to - # commit since there's no way to generate a correct Manifest. - # Do not try to do any more QA checks on this package since missing - # metadata leads to false positives for several checks, and false - # positives confuse users. - self.continue_ = True - can_force.set(False, ignore_InvalidState=True) - self.pkgs = pkgs - # set our updated data - dyn_pkgs = kwargs.get('pkgs') - dyn_pkgs.set(pkgs) - return self.continue_ - - @property - def runInPkgs(self): - '''Package level scans''' - return (True, [self.check_isebuild]) - - @property - def runInEbuilds(self): - '''Ebuild level scans''' - return (True, [self.check, self.set_pkg_data, self.bad_split_check, self.pkg_invalid]) diff --git a/repoman/pym/repoman/modules/scan/ebuild/multicheck.py b/repoman/pym/repoman/modules/scan/ebuild/multicheck.py deleted file mode 100644 index 94526ae9e..000000000 --- a/repoman/pym/repoman/modules/scan/ebuild/multicheck.py +++ /dev/null @@ -1,60 +0,0 @@ - -'''multicheck.py -Perform multiple different checks on an ebuild -''' - -import io - -from portage import _encodings, _unicode_encode - -from repoman.modules.scan.scanbase import ScanBase -from repoman.modules.linechecks.controller import LineCheckController - - -class MultiCheck(ScanBase): - '''Class to run multiple different checks on an ebuild''' - - def __init__(self, **kwargs): - '''Class init - - @param qatracker: QATracker instance - @param options: the run time cli options - ''' - self.qatracker = kwargs.get('qatracker') - self.options = kwargs.get('options') - self.controller = LineCheckController( - kwargs.get('repo_settings'), - kwargs.get('linechecks') - ) - self.controller.checks_init(self.options.experimental_inherit == 'y') - - def check(self, **kwargs): - '''Check the ebuild for utf-8 encoding - - @param pkg: Package in which we check (object). - @param ebuild: Ebuild which we check (object). - @returns: dictionary - ''' - ebuild = kwargs.get('ebuild').get() - pkg = kwargs.get('pkg').get() - try: - # All ebuilds should have utf_8 encoding. - f = io.open( - _unicode_encode(ebuild.full_path, encoding=_encodings['fs'], - errors='strict'), - mode='r', encoding=_encodings['repo.content']) - try: - for check_name, e in self.controller.run_checks(f, pkg): - self.qatracker.add_error( - check_name, ebuild.relative_path + ': %s' % e) - finally: - f.close() - except UnicodeDecodeError: - # A file.UTF8 failure will have already been recorded. - pass - return False - - @property - def runInEbuilds(self): - '''Ebuild level scans''' - return (True, [self.check]) diff --git a/repoman/pym/repoman/modules/scan/eclasses/__init__.py b/repoman/pym/repoman/modules/scan/eclasses/__init__.py deleted file mode 100644 index 234ef9fca..000000000 --- a/repoman/pym/repoman/modules/scan/eclasses/__init__.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Eclasses plug-in module for repoman. -Performs an live and ruby eclass checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'eclasses', - 'description': doc, - 'provides':{ - 'live-module': { - 'name': "live", - 'sourcefile': "live", - 'class': "LiveEclassChecks", - 'description': doc, - 'functions': ['check'], - 'func_kwargs': { - }, - 'mod_kwargs': ['qatracker', 'repo_metadata', 'repo_settings', - ], - 'func_kwargs': { - 'ebuild': (None, None), - 'pkg': (None, None), - 'xpkg': (None, None), - 'y_ebuild': (None, None), - }, - 'module_runsIn': ['ebuilds'], - }, - 'ruby-module': { - 'name': "ruby", - 'sourcefile': "ruby", - 'class': "RubyEclassChecks", - 'description': doc, - 'functions': ['check'], - 'func_kwargs': { - }, - 'mod_kwargs': ['qatracker', 'repo_settings' - ], - 'func_kwargs': { - 'ebuild': (None, None), - 'pkg': (None, None), - }, - 'module_runsIn': ['ebuilds'], - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/scan/eclasses/live.py b/repoman/pym/repoman/modules/scan/eclasses/live.py deleted file mode 100644 index dca10b583..000000000 --- a/repoman/pym/repoman/modules/scan/eclasses/live.py +++ /dev/null @@ -1,76 +0,0 @@ - -'''live.py -Performs Live eclass checks -''' - -from repoman._portage import portage -from repoman.modules.scan.scanbase import ScanBase - - -class LiveEclassChecks(ScanBase): - '''Performs checks for the usage of Live eclasses in ebuilds''' - - def __init__(self, **kwargs): - ''' - @param qatracker: QATracker instance - ''' - self.qatracker = kwargs.get('qatracker') - self.pmaskdict = kwargs.get('repo_metadata')['pmaskdict'] - self.repo_settings = kwargs.get('repo_settings') - - def check(self, **kwargs): - '''Ebuilds that inherit a "Live" eclass (darcs, subversion, git, cvs, - etc..) should not be allowed to be marked stable - - @param pkg: Package in which we check (object). - @param xpkg: Package in which we check (string). - @param ebuild: Ebuild which we check (object). - @param y_ebuild: Ebuild which we check (string). - @returns: boolean - ''' - pkg = kwargs.get("pkg").result() - package = kwargs.get('xpkg') - ebuild = kwargs.get('ebuild').get() - y_ebuild = kwargs.get('y_ebuild') - - if ebuild.live_ebuild and self.repo_settings.repo_config.name == "gentoo": - return self.check_live(pkg, package, ebuild, y_ebuild) - return False - - def check_live(self, pkg, package, ebuild, y_ebuild): - '''Perform the live vcs check - - @param pkg: Package in which we check (object). - @param xpkg: Package in which we check (string). - @param ebuild: Ebuild which we check (object). - @param y_ebuild: Ebuild which we check (string). - @returns: boolean - ''' - keywords = ebuild.keywords - is_stable = lambda kw: not kw.startswith("~") and not kw.startswith("-") - bad_stable_keywords = list(filter(is_stable, keywords)) - - if bad_stable_keywords: - self.qatracker.add_error( - "LIVEVCS.stable", "%s/%s.ebuild with stable keywords: %s" % ( - package, y_ebuild, bad_stable_keywords)) - - good_keywords_exist = len(bad_stable_keywords) < len(keywords) - if good_keywords_exist and not self._has_global_mask(pkg, self.pmaskdict): - self.qatracker.add_error("LIVEVCS.unmasked", ebuild.relative_path) - return False - - @staticmethod - def _has_global_mask(pkg, global_pmaskdict): - mask_atoms = global_pmaskdict.get(pkg.cp) - if mask_atoms: - pkg_list = [pkg] - for x in mask_atoms: - if portage.dep.match_from_list(x, pkg_list): - return x - return None - - @property - def runInEbuilds(self): - '''Ebuild level scans''' - return (True, [self.check]) diff --git a/repoman/pym/repoman/modules/scan/eclasses/ruby.py b/repoman/pym/repoman/modules/scan/eclasses/ruby.py deleted file mode 100644 index c5adeedf1..000000000 --- a/repoman/pym/repoman/modules/scan/eclasses/ruby.py +++ /dev/null @@ -1,49 +0,0 @@ - -'''ruby.py -Performs Ruby eclass checks -''' - -from repoman.modules.scan.scanbase import ScanBase - - -class RubyEclassChecks(ScanBase): - '''Performs checks for the usage of Ruby eclasses in ebuilds''' - - def __init__(self, **kwargs): - ''' - @param qatracker: QATracker instance - ''' - super(RubyEclassChecks, self).__init__(**kwargs) - self.qatracker = kwargs.get('qatracker') - self.repo_settings = kwargs.get('repo_settings') - self.old_ruby_eclasses = ["ruby-ng", "ruby-fakegem", "ruby"] - - def check(self, **kwargs): - '''Check ebuilds that inherit the ruby eclasses - - @param pkg: Package in which we check (object). - @param ebuild: Ebuild which we check (object). - @returns: dictionary - ''' - pkg = kwargs.get('pkg').get() - ebuild = kwargs.get('ebuild').get() - is_inherited = lambda eclass: eclass in pkg.inherited - is_old_ruby_eclass_inherited = filter( - is_inherited, self.old_ruby_eclasses) - - if is_old_ruby_eclass_inherited: - ruby_intersection = pkg.iuse.all.intersection( - self.repo_settings.qadata.ruby_deprecated) - - if ruby_intersection: - for myruby in ruby_intersection: - self.qatracker.add_error( - "IUSE.rubydeprecated", - (ebuild.relative_path + ": Deprecated ruby target: %s") - % myruby) - return False - - @property - def runInEbuilds(self): - '''Ebuild level scans''' - return (True, [self.check]) diff --git a/repoman/pym/repoman/modules/scan/fetch/__init__.py b/repoman/pym/repoman/modules/scan/fetch/__init__.py deleted file mode 100644 index 30db3f529..000000000 --- a/repoman/pym/repoman/modules/scan/fetch/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """fetches plug-in module for repoman. -Performs fetch related checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'fetches', - 'description': doc, - 'provides':{ - 'fetches-module': { - 'name': "fetches", - 'sourcefile': "fetches", - 'class': "FetchChecks", - 'description': doc, - 'functions': ['check'], - 'func_desc': { - }, - 'mod_kwargs': ['portdb', 'qatracker', 'repo_settings', 'vcs_settings', - ], - 'func_kwargs': { - 'changed': (None, None), - 'checkdir': (None, None), - 'checkdir_relative': (None, None), - 'ebuild': (None, None), - 'xpkg': (None, None), - }, - 'module_runsIn': ['pkgs', 'ebuilds'], - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/scan/fetch/fetches.py b/repoman/pym/repoman/modules/scan/fetch/fetches.py deleted file mode 100644 index 5a958a461..000000000 --- a/repoman/pym/repoman/modules/scan/fetch/fetches.py +++ /dev/null @@ -1,199 +0,0 @@ -# -*- coding:utf-8 -*- - -'''fetches.py -Performs the src_uri fetchlist and files checks -''' - -from stat import S_ISDIR - -# import our initialized portage instance -from repoman._portage import portage -from repoman.modules.vcs.vcs import vcs_new_changed -from repoman.modules.scan.scanbase import ScanBase - -from portage import os - - -class FetchChecks(ScanBase): - '''Performs checks on the files needed for the ebuild''' - - def __init__(self, **kwargs): - ''' - @param portdb: portdb instance - @param qatracker: QATracker instance - @param repo_settings: repository settings instance - @param vcs_settings: VCSSettings instance - ''' - super(FetchChecks, self).__init__(**kwargs) - self.portdb = kwargs.get('portdb') - self.qatracker = kwargs.get('qatracker') - self.repo_settings = kwargs.get('repo_settings') - self.repoman_settings = self.repo_settings.repoman_settings - self.vcs_settings = kwargs.get('vcs_settings') - self._src_uri_error = False - - # TODO: Build a regex instead here, for the SRC_URI.mirror check. - self.thirdpartymirrors = {} - profile_thirdpartymirrors = self.repo_settings.repoman_settings.thirdpartymirrors().items() - for mirror_alias, mirrors in profile_thirdpartymirrors: - # Skip thirdpartymirrors that do not list more than one mirror - # anymore. There is no point in using mirror:// there and this - # means that the thirdpartymirrors entry will most likely - # be removed anyway. - if len(mirrors) <= 1: - continue - for mirror in mirrors: - if not mirror.endswith("/"): - mirror += "/" - self.thirdpartymirrors[mirror] = mirror_alias - - def check(self, **kwargs): - '''Checks the ebuild sources and files for errors - - @param xpkg: the pacakge being checked - @param checkdir: string, directory path - @param checkdir_relative: repolevel determined path - @returns: boolean - ''' - xpkg = kwargs.get('xpkg') - checkdir = kwargs.get('checkdir') - checkdir_relative = kwargs.get('checkdir_relative') - changed = kwargs.get('changed').changed - new = kwargs.get('changed').new - _digests = self.digests(checkdir) - fetchlist_dict = portage.FetchlistDict( - checkdir, self.repoman_settings, self.portdb) - myfiles_all = [] - self._src_uri_error = False - for mykey in fetchlist_dict: - try: - myfiles_all.extend(fetchlist_dict[mykey]) - except portage.exception.InvalidDependString as e: - self._src_uri_error = True - try: - self.portdb.aux_get(mykey, ["SRC_URI"]) - except KeyError: - # This will be reported as an "ebuild.syntax" error. - pass - else: - self.qatracker.add_error( - "SRC_URI.syntax", "%s.ebuild SRC_URI: %s" % (mykey, e)) - del fetchlist_dict - if not self._src_uri_error: - # This test can produce false positives if SRC_URI could not - # be parsed for one or more ebuilds. There's no point in - # producing a false error here since the root cause will - # produce a valid error elsewhere, such as "SRC_URI.syntax" - # or "ebuild.sytax". - myfiles_all = set(myfiles_all) - for entry in _digests: - if entry not in myfiles_all: - self.qatracker.add_error("digest.unused", checkdir + "::" + entry) - for entry in myfiles_all: - if entry not in _digests: - self.qatracker.add_error("digest.missing", checkdir + "::" + entry) - del myfiles_all - - if os.path.exists(checkdir + "/files"): - filesdirlist = os.listdir(checkdir + "/files") - - # Recurse through files directory, use filesdirlist as a stack; - # appending directories as needed, - # so people can't hide > 20k files in a subdirectory. - while filesdirlist: - y = filesdirlist.pop(0) - relative_path = os.path.join(xpkg, "files", y) - full_path = os.path.join(self.repo_settings.repodir, relative_path) - try: - mystat = os.stat(full_path) - except OSError as oe: - if oe.errno == 2: - # don't worry about it. it likely was removed via fix above. - continue - else: - raise oe - if S_ISDIR(mystat.st_mode): - if self.vcs_settings.status.isVcsDir(y): - continue - for z in os.listdir(checkdir + "/files/" + y): - if self.vcs_settings.status.isVcsDir(z): - continue - filesdirlist.append(y + "/" + z) - # Current policy is no files over 20 KiB, these are the checks. - # File size between 20 KiB and 60 KiB causes a warning, - # while file size over 60 KiB causes an error. - elif mystat.st_size > 61440: - self.qatracker.add_error( - "file.size-fatal", "(%d KiB) %s/files/%s" % ( - mystat.st_size // 1024, xpkg, y)) - elif mystat.st_size > 20480: - self.qatracker.add_error( - "file.size", "(%d KiB) %s/files/%s" % ( - mystat.st_size // 1024, xpkg, y)) - elif mystat.st_size == 0: - self.qatracker.add_error( - "file.empty", "%s/files/%s" % (xpkg, y)) - - index = self.repo_settings.repo_config.find_invalid_path_char(y) - if index != -1: - y_relative = os.path.join(checkdir_relative, "files", y) - if self.vcs_settings.vcs is not None \ - and not vcs_new_changed(y_relative, changed, new): - # If the file isn't in the VCS new or changed set, then - # assume that it's an irrelevant temporary file (Manifest - # entries are not generated for file names containing - # prohibited characters). See bug #406877. - index = -1 - if index != -1: - self.qatracker.add_error( - "file.name", - "%s/files/%s: char '%s'" % (checkdir, y, y[index])) - return False - - def digests(self, checkdir): - '''Returns the freshly loaded digests - - @param checkdir: string, directory path - ''' - mf = self.repoman_settings.repositories.get_repo_for_location( - os.path.dirname(os.path.dirname(checkdir))) - mf = mf.load_manifest(checkdir, self.repoman_settings["DISTDIR"]) - _digests = mf.getTypeDigests("DIST") - del mf - return _digests - - def check_mirrors(self, **kwargs): - '''Check that URIs don't reference a server from thirdpartymirrors - - @param ebuild: Ebuild which we check (object). - @returns: boolean - ''' - ebuild = kwargs.get('ebuild').get() - - for uri in portage.dep.use_reduce( - ebuild.metadata["SRC_URI"], matchall=True, is_src_uri=True, - eapi=ebuild.eapi, flat=True): - contains_mirror = False - for mirror, mirror_alias in self.thirdpartymirrors.items(): - if uri.startswith(mirror): - contains_mirror = True - break - if not contains_mirror: - continue - - new_uri = "mirror://%s/%s" % (mirror_alias, uri[len(mirror):]) - self.qatracker.add_error( - "SRC_URI.mirror", - "%s: '%s' found in thirdpartymirrors, use '%s'" % ( - ebuild.relative_path, mirror, new_uri)) - return False - - @property - def runInPkgs(self): - '''Package level scans''' - return (True, [self.check]) - - @property - def runInEbuilds(self): - '''Ebuild level scans''' - return (True, [self.check_mirrors]) diff --git a/repoman/pym/repoman/modules/scan/keywords/__init__.py b/repoman/pym/repoman/modules/scan/keywords/__init__.py deleted file mode 100644 index d4c1281bc..000000000 --- a/repoman/pym/repoman/modules/scan/keywords/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Keywords plug-in module for repoman. -Performs keywords checks on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'keywords', - 'description': doc, - 'provides':{ - 'keywords-module': { - 'name': "keywords", - 'sourcefile': "keywords", - 'class': "KeywordChecks", - 'description': doc, - 'functions': ['prepare', 'check'], - 'func_desc': { - }, - 'mod_kwargs': ['qatracker', 'options', 'repo_metadata', 'profiles', - ], - 'func_kwargs': { - 'changed': (None, None), - 'ebuild': ('Future', 'UNSET'), - 'pkg': ('Future', 'UNSET'), - 'xpkg': None, - 'y_ebuild': (None, None), - }, - 'module_runsIn': ['pkgs', 'ebuilds', 'final'], - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/scan/keywords/keywords.py b/repoman/pym/repoman/modules/scan/keywords/keywords.py deleted file mode 100644 index 556f352ad..000000000 --- a/repoman/pym/repoman/modules/scan/keywords/keywords.py +++ /dev/null @@ -1,170 +0,0 @@ -# -*- coding:utf-8 -*- - -'''keywords.py -Perform KEYWORDS related checks - -''' - -from repoman.modules.scan.scanbase import ScanBase - - -class KeywordChecks(ScanBase): - '''Perform checks on the KEYWORDS of an ebuild''' - - def __init__(self, **kwargs): - ''' - @param qatracker: QATracker instance - @param options: argparse options instance - ''' - super(KeywordChecks, self).__init__(**kwargs) - self.qatracker = kwargs.get('qatracker') - self.options = kwargs.get('options') - self.repo_metadata = kwargs.get('repo_metadata') - self.profiles = kwargs.get('profiles') - self.slot_keywords = {} - - def prepare(self, **kwargs): - '''Prepare the checks for the next package.''' - self.slot_keywords = {} - self.dropped_keywords = {} - return False - - def check(self, **kwargs): - '''Perform the check. - - @param pkg: Package in which we check (object). - @param xpkg: Package in which we check (string). - @param ebuild: Ebuild which we check (object). - @param y_ebuild: Ebuild which we check (string). - @param ebuild_archs: Just the architectures (no prefixes) of the ebuild. - @param changed: Changes instance - @returns: dictionary - ''' - pkg = kwargs.get('pkg').get() - xpkg =kwargs.get('xpkg') - ebuild = kwargs.get('ebuild').get() - y_ebuild = kwargs.get('y_ebuild') - changed = kwargs.get('changed') - if not self.options.straight_to_stable: - self._checkAddedWithStableKeywords( - xpkg, ebuild, y_ebuild, ebuild.keywords, changed) - - self._checkForDroppedKeywords(pkg, ebuild, ebuild.archs) - - self._checkForInvalidKeywords(ebuild, xpkg, y_ebuild) - - self._checkForMaskLikeKeywords(xpkg, y_ebuild, ebuild.keywords) - - self._checkForUnsortedKeywords(ebuild, xpkg, y_ebuild) - - self.slot_keywords[pkg.slot].update(ebuild.archs) - return False - - def check_dropped_keywords(self, **kwargs): - '''Report on any dropped keywords for the latest ebuild in a slot - - @returns: boolean - ''' - for ebuild, arches in self.dropped_keywords.values(): - if arches: - self.qatracker.add_error( - "KEYWORDS.dropped", "%s: %s" % ( - ebuild, - " ".join(sorted(arches)))) - return False - - @staticmethod - def _isKeywordStable(keyword): - return not keyword.startswith("~") and not keyword.startswith("-") - - def _checkAddedWithStableKeywords( - self, package, ebuild, y_ebuild, keywords, changed): - catdir, pkgdir = package.split("/") - - stable_keywords = list(filter(self._isKeywordStable, keywords)) - if stable_keywords: - if ebuild.ebuild_path in changed.new_ebuilds and catdir != "virtual": - stable_keywords.sort() - self.qatracker.add_error( - "KEYWORDS.stable", - "%s/%s.ebuild added with stable keywords: %s" % - (package, y_ebuild, " ".join(stable_keywords))) - - def _checkForDroppedKeywords( - self, pkg, ebuild, ebuild_archs): - previous_keywords = self.slot_keywords.get(pkg.slot) - if previous_keywords is None: - self.slot_keywords[pkg.slot] = set() - elif ebuild_archs and "*" not in ebuild_archs and not ebuild.live_ebuild: - self.slot_keywords[pkg.slot].update(ebuild_archs) - dropped_keywords = previous_keywords.difference(ebuild_archs) - self.dropped_keywords[pkg.slot] = (ebuild.relative_path, { arch for arch in dropped_keywords}) - - def _checkForInvalidKeywords(self, ebuild, xpkg, y_ebuild): - myuse = ebuild.keywords - - for mykey in myuse: - if mykey not in ("-*", "*", "~*"): - myskey = mykey - - if not self._isKeywordStable(myskey[:1]): - myskey = myskey[1:] - - if myskey not in self.repo_metadata['kwlist']: - self.qatracker.add_error("KEYWORDS.invalid", - "%s/%s.ebuild: %s" % (xpkg, y_ebuild, mykey)) - elif myskey not in self.profiles: - self.qatracker.add_error( - "KEYWORDS.invalid", - "%s/%s.ebuild: %s (profile invalid)" - % (xpkg, y_ebuild, mykey)) - - def _checkForMaskLikeKeywords(self, xpkg, y_ebuild, keywords): - # KEYWORDS="-*" is a stupid replacement for package.mask - # and screws general KEYWORDS semantics - if "-*" in keywords: - haskeyword = False - - for kw in keywords: - if kw[0] == "~": - kw = kw[1:] - if kw in self.repo_metadata['kwlist']: - haskeyword = True - - if not haskeyword: - self.qatracker.add_error("KEYWORDS.stupid", - "%s/%s.ebuild" % (xpkg, y_ebuild)) - - def _checkForUnsortedKeywords(self, ebuild, xpkg, y_ebuild): - '''Ebuilds that contain KEYWORDS - which are not sorted alphabetically.''' - def sort_keywords(kw): - # Split keywords containing hyphens. The part after - # the hyphen should be treated as the primary key. - # This is consistent with ekeyword. - parts = list(reversed(kw.lstrip("~-").split("-", 1))) - # Hack to make sure that keywords - # without hyphens are sorted first - if len(parts) == 1: - parts.insert(0, "") - return parts - sorted_keywords = sorted(ebuild.keywords, key=sort_keywords) - if sorted_keywords != ebuild.keywords: - self.qatracker.add_error("KEYWORDS.unsorted", - "%s/%s.ebuild contains unsorted keywords" % - (xpkg, y_ebuild)) - - @property - def runInPkgs(self): - '''Package level scans''' - return (True, [self.prepare]) - - @property - def runInEbuilds(self): - '''Ebuild level scans''' - return (True, [self.check]) - - @property - def runInFinal(self): - '''Final package level scans''' - return (True, [self.check_dropped_keywords]) diff --git a/repoman/pym/repoman/modules/scan/manifest/__init__.py b/repoman/pym/repoman/modules/scan/manifest/__init__.py deleted file mode 100644 index 9e05c5288..000000000 --- a/repoman/pym/repoman/modules/scan/manifest/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Ebuild plug-in module for repoman. -Performs an IsEbuild check on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'manifest', - 'description': doc, - 'provides':{ - 'manifest-module': { - 'name': "manifests", - 'sourcefile': "manifests", - 'class': "Manifests", - 'description': doc, - 'functions': ['check', 'create_manifest', 'digest_check'], - 'func_desc': { - }, - 'mod_kwargs': ['options', 'portdb', 'qatracker', 'repo_settings', - ], - 'func_kwargs': { - 'checkdir': (None, None), - 'xpkg': (None, None), - }, - 'module_runsIn': ['pkgs'], - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/scan/manifest/manifests.py b/repoman/pym/repoman/modules/scan/manifest/manifests.py deleted file mode 100644 index dc2b338e4..000000000 --- a/repoman/pym/repoman/modules/scan/manifest/manifests.py +++ /dev/null @@ -1,56 +0,0 @@ -# -*- coding:utf-8 -*- - -# import our initialized portage instance -from repoman._portage import portage -from repoman.modules.scan.scanbase import ScanBase - -from portage import os - - -class Manifests(ScanBase): - '''Creates as well as checks pkg Manifest entries/files''' - - def __init__(self, **kwargs): - '''Class init - - @param options: the run time cli options - @param portdb: portdb instance - @param qatracker: QATracker instance - @param repo_settings: repository settings instance - ''' - self.options = kwargs.get('options') - self.portdb = kwargs.get('portdb') - self.qatracker = kwargs.get('qatracker') - self.repoman_settings = kwargs.get('repo_settings').repoman_settings - - def check(self, **kwargs): - '''Perform a changelog and untracked checks on the ebuild - - @param xpkg: Package in which we check (object). - @param checkdir: the current package directory - @returns: dictionary - ''' - checkdir = kwargs.get('checkdir') - xpkg = kwargs.get('xpkg') - if self.options.pretend: - return False - self.digest_check(xpkg, checkdir) - if self.options.mode == 'manifest-check': - return True - return False - - def digest_check(self, xpkg, checkdir): - '''Check the manifest entries, report any Q/A errors - - @param xpkg: the cat/pkg name to check - @param checkdir: the directory path to check''' - self.repoman_settings['O'] = checkdir - self.repoman_settings['PORTAGE_QUIET'] = '1' - if not portage.digestcheck([], self.repoman_settings, strict=1): - self.qatracker.add_error("manifest.bad", os.path.join(xpkg, 'Manifest')) - self.repoman_settings.pop('PORTAGE_QUIET', None) - - @property - def runInPkgs(self): - '''Package level scans''' - return (True, [self.check]) diff --git a/repoman/pym/repoman/modules/scan/metadata/__init__.py b/repoman/pym/repoman/modules/scan/metadata/__init__.py deleted file mode 100644 index fccde99b5..000000000 --- a/repoman/pym/repoman/modules/scan/metadata/__init__.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Metadata plug-in module for repoman. -Performs metadata checks on packages.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'metadata', - 'description': doc, - 'provides':{ - 'pkg-metadata': { - 'name': "pkgmetadata", - 'sourcefile': "pkgmetadata", - 'class': "PkgMetadata", - 'description': doc, - 'functions': ['check'], - 'func_desc': { - }, - 'mod_kwargs': ['repo_settings', 'qatracker', 'options', - 'metadata_xsd', 'uselist', - ], - 'func_kwargs': { - 'checkdir': (None, None), - 'checkdirlist': (None, None), - 'ebuild': (None, None), - 'pkg': (None, None), - 'repolevel': (None, None), - 'validity_future': (None, None), - 'xpkg': (None, None), - 'y_ebuild': (None, None), - }, - 'module_runsIn': ['pkgs', 'ebuilds', 'final'], - }, - 'ebuild-metadata': { - 'name': "ebuild_metadata", - 'sourcefile': "ebuild_metadata", - 'class': "EbuildMetadata", - 'description': doc, - 'functions': ['check'], - 'func_desc': { - }, - 'mod_kwargs': ['qatracker', 'repo_settings', - ], - 'func_kwargs': { - 'catdir': (None, None), - 'ebuild': (None, None), - 'xpkg': (None, None), - 'y_ebuild': (None, None), - }, - 'module_runsIn': ['ebuilds'], - }, - 'description-metadata': { - 'name': "description", - 'sourcefile': "description", - 'class': "DescriptionChecks", - 'description': doc, - 'functions': ['check'], - 'func_desc': { - }, - 'mod_kwargs': ['qatracker', 'repo_settings' - ], - 'func_kwargs': { - 'ebuild': (None, None), - 'pkg': ('Future', 'UNSET'), - }, - 'module_runsIn': ['ebuilds'], - }, - 'restrict-metadata': { - 'name': "restrict", - 'sourcefile': "restrict", - 'class': "RestrictChecks", - 'description': doc, - 'functions': ['check'], - 'func_desc': { - }, - 'mod_kwargs': ['qatracker', 'repo_settings' - ], - 'func_kwargs': { - 'ebuild': (None, None), - 'xpkg': (None, None), - 'y_ebuild': (None, None), - }, - 'module_runsIn': ['ebuilds'], - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/scan/metadata/description.py b/repoman/pym/repoman/modules/scan/metadata/description.py deleted file mode 100644 index 0ce81a2dc..000000000 --- a/repoman/pym/repoman/modules/scan/metadata/description.py +++ /dev/null @@ -1,41 +0,0 @@ - -'''description.py -Perform checks on the DESCRIPTION variable. -''' - -from repoman.modules.scan.scanbase import ScanBase - - -class DescriptionChecks(ScanBase): - '''Perform checks on the DESCRIPTION variable.''' - - def __init__(self, **kwargs): - ''' - @param qatracker: QATracker instance - ''' - self.qatracker = kwargs.get('qatracker') - self.repo_settings = kwargs.get('repo_settings') - - def checkTooLong(self, **kwargs): - ''' - @param pkg: Package in which we check (object). - @param ebuild: Ebuild which we check (object). - ''' - ebuild = kwargs.get('ebuild').get() - pkg = kwargs.get('pkg').get() - # 14 is the length of DESCRIPTION="" - if len(pkg._metadata['DESCRIPTION']) > self.repo_settings.qadata.max_desc_len: - self.qatracker.add_error( - 'DESCRIPTION.toolong', - "%s: DESCRIPTION is %d characters (max %d)" % - (ebuild.relative_path, len( - pkg._metadata['DESCRIPTION']), self.repo_settings.qadata.max_desc_len)) - return False - - @property - def runInPkgs(self): - return (False, []) - - @property - def runInEbuilds(self): - return (True, [self.checkTooLong]) diff --git a/repoman/pym/repoman/modules/scan/metadata/ebuild_metadata.py b/repoman/pym/repoman/modules/scan/metadata/ebuild_metadata.py deleted file mode 100644 index 4c35bbc12..000000000 --- a/repoman/pym/repoman/modules/scan/metadata/ebuild_metadata.py +++ /dev/null @@ -1,78 +0,0 @@ -# -*- coding:utf-8 -*- - -'''Ebuild Metadata Checks''' - -import re -import sys - -if sys.hexversion >= 0x3000000: - basestring = str - -from repoman.modules.scan.scanbase import ScanBase - -from portage.dep import use_reduce - -NON_ASCII_RE = re.compile(r'[^\x00-\x7f]') -URISCHEME_RE = re.compile(r'^[a-z][0-9a-z\-\.\+]+://') - -class EbuildMetadata(ScanBase): - - def __init__(self, **kwargs): - self.qatracker = kwargs.get('qatracker') - self.repo_settings = kwargs.get('repo_settings') - - def invalidchar(self, **kwargs): - ebuild = kwargs.get('ebuild').get() - for k, v in ebuild.metadata.items(): - if not isinstance(v, basestring): - continue - m = NON_ASCII_RE.search(v) - if m is not None: - self.qatracker.add_error( - "variable.invalidchar", - "%s: %s variable contains non-ASCII " - "character at position %s" % - (ebuild.relative_path, k, m.start() + 1)) - return False - - def missing(self, **kwargs): - ebuild = kwargs.get('ebuild').get() - for pos, missing_var in enumerate(self.repo_settings.qadata.missingvars): - if not ebuild.metadata.get(missing_var): - if kwargs.get('catdir') == "virtual" and \ - missing_var in ("HOMEPAGE", "LICENSE"): - continue - if ebuild.live_ebuild and missing_var == "KEYWORDS": - continue - myqakey = self.repo_settings.qadata.missingvars[pos] + ".missing" - self.qatracker.add_error(myqakey, '%s/%s.ebuild' - % (kwargs.get('xpkg'), kwargs.get('y_ebuild'))) - return False - - def virtual(self, **kwargs): - ebuild = kwargs.get('ebuild').get() - if kwargs.get('catdir') == "virtual": - for var in ("HOMEPAGE", "LICENSE"): - if ebuild.metadata.get(var): - myqakey = var + ".virtual" - self.qatracker.add_error(myqakey, ebuild.relative_path) - return False - - def homepage_urischeme(self, **kwargs): - ebuild = kwargs.get('ebuild').get() - if kwargs.get('catdir') != "virtual": - for homepage in use_reduce(ebuild.metadata["HOMEPAGE"], - matchall=True,flat=True): - if URISCHEME_RE.match(homepage) is None: - self.qatracker.add_error( - "HOMEPAGE.missingurischeme", ebuild.relative_path) - return False - - @property - def runInPkgs(self): - return (False, []) - - @property - def runInEbuilds(self): - return (True, [self.invalidchar, self.missing, - self.virtual, self.homepage_urischeme]) diff --git a/repoman/pym/repoman/modules/scan/metadata/pkgmetadata.py b/repoman/pym/repoman/modules/scan/metadata/pkgmetadata.py deleted file mode 100644 index 3c38697fe..000000000 --- a/repoman/pym/repoman/modules/scan/metadata/pkgmetadata.py +++ /dev/null @@ -1,208 +0,0 @@ -# -*- coding:utf-8 -*- - -'''Package Metadata Checks operations''' - -import sys -import re - -from itertools import chain -from collections import Counter - -try: - from lxml import etree - from lxml.etree import ParserError -except (SystemExit, KeyboardInterrupt): - raise -except (ImportError, SystemError, RuntimeError, Exception): - # broken or missing xml support - # https://bugs.python.org/issue14988 - msg = ["Please emerge dev-python/lxml in order to use repoman."] - from portage.output import EOutput - out = EOutput() - for line in msg: - out.eerror(line) - sys.exit(1) - -# import our initialized portage instance -from repoman._portage import portage -from repoman.metadata import metadata_dtd_uri -from repoman.modules.scan.scanbase import ScanBase - -from portage.exception import InvalidAtom -from portage import os -from portage.dep import Atom -from portage.xml.metadata import parse_metadata_use - -from .use_flags import USEFlagChecks - -metadata_xml_encoding = 'UTF-8' -metadata_xml_declaration = '<?xml version="1.0" encoding="%s"?>' \ - % (metadata_xml_encoding,) -metadata_doctype_name = 'pkgmetadata' - - -class PkgMetadata(ScanBase, USEFlagChecks): - '''Package metadata.xml checks''' - - def __init__(self, **kwargs): - '''PkgMetadata init function - - @param repo_settings: settings instance - @param qatracker: QATracker instance - @param options: argparse options instance - @param metadata_xsd: path of metadata.xsd - ''' - super(PkgMetadata, self).__init__(**kwargs) - repo_settings = kwargs.get('repo_settings') - self.qatracker = kwargs.get('qatracker') - self.options = kwargs.get('options') - self.metadata_xsd = kwargs.get('metadata_xsd') - self.globalUseFlags = kwargs.get('uselist') - self.repoman_settings = repo_settings.repoman_settings - self.musedict = {} - self.muselist = set() - - def check(self, **kwargs): - '''Performs the checks on the metadata.xml for the package - @param xpkg: the pacakge being checked - @param checkdir: string, directory path - @param checkdirlist: list of checkdir's - @param repolevel: integer - @returns: boolean - ''' - xpkg = kwargs.get('xpkg') - checkdir = kwargs.get('checkdir') - checkdirlist = kwargs.get('checkdirlist').get() - - self.musedict = {} - if self.options.mode in ['manifest']: - self.muselist = frozenset(self.musedict) - return False - - # metadata.xml file check - if "metadata.xml" not in checkdirlist: - self.qatracker.add_error("metadata.missing", xpkg + "/metadata.xml") - self.muselist = frozenset(self.musedict) - return False - - # metadata.xml parse check - metadata_bad = False - - # read metadata.xml into memory - try: - _metadata_xml = etree.parse(os.path.join(checkdir, 'metadata.xml')) - except (ParserError, SyntaxError, EnvironmentError) as e: - metadata_bad = True - self.qatracker.add_error("metadata.bad", "%s/metadata.xml: %s" % (xpkg, e)) - del e - self.muselist = frozenset(self.musedict) - return False - - indentation_chars = Counter() - for l in etree.tostring(_metadata_xml).splitlines(): - indentation_chars.update(re.match(b"\s*", l).group(0)) - if len(indentation_chars) > 1: - self.qatracker.add_error("metadata.warning", "%s/metadata.xml: %s" % - (xpkg, "inconsistent use of tabs and spaces in indentation") - ) - - xml_encoding = _metadata_xml.docinfo.encoding - if xml_encoding.upper() != metadata_xml_encoding: - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: " - "xml declaration encoding should be '%s', not '%s'" % - (xpkg, metadata_xml_encoding, xml_encoding)) - - if not _metadata_xml.docinfo.doctype: - metadata_bad = True - self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: %s" % (xpkg, "DOCTYPE is missing")) - else: - doctype_system = _metadata_xml.docinfo.system_url - if doctype_system.replace('http://', 'https://') != metadata_dtd_uri: - if doctype_system is None: - system_problem = "but it is undefined" - else: - system_problem = "not '%s'" % doctype_system - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: " - "DOCTYPE: SYSTEM should refer to '%s', %s" % - (xpkg, metadata_dtd_uri, system_problem)) - doctype_name = _metadata_xml.docinfo.doctype.split(' ')[1] - if doctype_name != metadata_doctype_name: - self.qatracker.add_error( - "metadata.bad", "%s/metadata.xml: " - "DOCTYPE: name should be '%s', not '%s'" % - (xpkg, metadata_doctype_name, doctype_name)) - - # load USE flags from metadata.xml - self.musedict = parse_metadata_use(_metadata_xml) - for atom in chain(*self.musedict.values()): - if atom is None: - continue - try: - atom = Atom(atom) - except InvalidAtom as e: - self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: Invalid atom: %s" % (xpkg, e)) - else: - if atom.cp != xpkg: - self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: Atom contains " - "unexpected cat/pn: %s" % (xpkg, atom)) - - # Only carry out if in package directory or check forced - if not metadata_bad: - validator = etree.XMLSchema(file=self.metadata_xsd) - if not validator.validate(_metadata_xml): - self._add_validate_errors(xpkg, validator.error_log) - self.muselist = frozenset(self.musedict) - return False - - def check_unused(self, **kwargs): - '''Reports on any unused metadata.xml use descriptions - - @param xpkg: the pacakge being checked - @param used_useflags: use flag list - @param validity_future: Future instance - ''' - xpkg = kwargs.get('xpkg') - valid_state = kwargs.get('validity_future').get() - # check if there are unused local USE-descriptions in metadata.xml - # (unless there are any invalids, to avoid noise) - if valid_state: - for myflag in self.muselist.difference(self.usedUseFlags): - self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: unused local USE-description: '%s'" - % (xpkg, myflag)) - return False - - def _add_validate_errors(self, xpkg, log): - listed = set() - for error in log: - msg_prefix = error.message.split(":",1)[0] - info = "%s %s" % (error.line, msg_prefix) - if info not in listed: - listed.add(info) - self.qatracker.add_error( - "metadata.bad", - "%s/metadata.xml: line: %s, %s" - % (xpkg, error.line, error.message)) - - @property - def runInPkgs(self): - '''Package level scans''' - return (True, [self.check]) - - @property - def runInEbuilds(self): - return (True, [self.check_useflags]) - - @property - def runInFinal(self): - '''Final scans at the package level''' - return (True, [self.check_unused]) diff --git a/repoman/pym/repoman/modules/scan/metadata/restrict.py b/repoman/pym/repoman/modules/scan/metadata/restrict.py deleted file mode 100644 index 99784f231..000000000 --- a/repoman/pym/repoman/modules/scan/metadata/restrict.py +++ /dev/null @@ -1,53 +0,0 @@ - -'''restrict.py -Perform checks on the RESTRICT variable. -''' - -# import our initialized portage instance -from repoman._portage import portage - -from repoman.modules.scan.scanbase import ScanBase - - -class RestrictChecks(ScanBase): - '''Perform checks on the RESTRICT variable.''' - - def __init__(self, **kwargs): - ''' - @param qatracker: QATracker instance - ''' - self.qatracker = kwargs.get('qatracker') - self.repo_settings = kwargs.get('repo_settings') - - def check(self, **kwargs): - xpkg = kwargs.get('xpkg') - ebuild = kwargs.get('ebuild').get() - y_ebuild = kwargs.get('y_ebuild') - myrestrict = None - - try: - myrestrict = portage.dep.use_reduce( - ebuild.metadata["RESTRICT"], matchall=1, flat=True) - except portage.exception.InvalidDependString as e: - self.qatracker.add_error("RESTRICT.syntax", - "%s: RESTRICT: %s" % (ebuild.relative_path, e)) - del e - - if myrestrict: - myrestrict = set(myrestrict) - mybadrestrict = myrestrict.difference(self.repo_settings.qadata.valid_restrict) - - if mybadrestrict: - for mybad in mybadrestrict: - self.qatracker.add_error("RESTRICT.invalid", - "%s/%s.ebuild: %s" % (xpkg, y_ebuild, mybad)) - return False - - @property - def runInPkgs(self): - return (False, []) - - @property - def runInEbuilds(self): - return (True, [self.check]) - diff --git a/repoman/pym/repoman/modules/scan/metadata/use_flags.py b/repoman/pym/repoman/modules/scan/metadata/use_flags.py deleted file mode 100644 index 1738fd23e..000000000 --- a/repoman/pym/repoman/modules/scan/metadata/use_flags.py +++ /dev/null @@ -1,94 +0,0 @@ -# -*- coding:utf-8 -*- - -'''use_flags.py -Performs USE flag related checks -''' - -# import our centrally initialized portage instance -from repoman._portage import portage - -from portage import eapi -from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use - - -class USEFlagChecks(object): - '''Performs checks on USE flags listed in the ebuilds and metadata.xml''' - - def __init__(self, **kwargs): - '''Class init - - @param qatracker: QATracker instance - @param globalUseFlags: Global USE flags - ''' - super(USEFlagChecks, self).__init__() - self.qatracker = None - self.globalUseFlags = None - self.useFlags = [] - self.defaultUseFlags = [] - self.usedUseFlags = set() - - def check_useflags(self, **kwargs): - '''Perform the check. - - @param pkg: Package in which we check (object). - @param xpkg: Package in which we check (string). - @param ebuild: Ebuild which we check (object). - @param y_ebuild: Ebuild which we check (string). - @returns: dictionary, including {ebuild_UsedUseFlags, used_useflags} - ''' - pkg = kwargs.get('pkg').get() - package = kwargs.get('xpkg') - ebuild = kwargs.get('ebuild').get() - y_ebuild = kwargs.get('y_ebuild') - # reset state variables for the run - self.useFlags = [] - self.defaultUseFlags = [] - # perform the checks - self._checkGlobal(pkg) - self._checkMetadata(package, ebuild, y_ebuild, self.muselist) - self._checkRequiredUSE(pkg, ebuild) - return False - - - def _checkGlobal(self, pkg): - for myflag in pkg._metadata["IUSE"].split(): - flag_name = myflag.lstrip("+-") - self.usedUseFlags.add(flag_name) - if myflag != flag_name: - self.defaultUseFlags.append(myflag) - if flag_name not in self.globalUseFlags: - self.useFlags.append(flag_name) - - def _checkMetadata(self, package, ebuild, y_ebuild, localUseFlags): - for mypos in range(len(self.useFlags) - 1, -1, -1): - if self.useFlags[mypos] and (self.useFlags[mypos] in localUseFlags): - del self.useFlags[mypos] - - if self.defaultUseFlags and not eapi_has_iuse_defaults(eapi): - for myflag in self.defaultUseFlags: - self.qatracker.add_error( - 'EAPI.incompatible', "%s: IUSE defaults" - " not supported with EAPI='%s': '%s'" % ( - ebuild.relative_path, eapi, myflag)) - - for mypos in range(len(self.useFlags)): - self.qatracker.add_error( - "IUSE.invalid", - "%s/%s.ebuild: %s" % (package, y_ebuild, self.useFlags[mypos])) - - def _checkRequiredUSE(self, pkg, ebuild): - required_use = pkg._metadata["REQUIRED_USE"] - if required_use: - if not eapi_has_required_use(eapi): - self.qatracker.add_error( - 'EAPI.incompatible', "%s: REQUIRED_USE" - " not supported with EAPI='%s'" - % (ebuild.relative_path, eapi,)) - try: - portage.dep.check_required_use( - required_use, (), pkg.iuse.is_valid_flag, eapi=eapi) - except portage.exception.InvalidDependString as e: - self.qatracker.add_error( - "REQUIRED_USE.syntax", - "%s: REQUIRED_USE: %s" % (ebuild.relative_path, e)) - del e diff --git a/repoman/pym/repoman/modules/scan/module.py b/repoman/pym/repoman/modules/scan/module.py deleted file mode 100644 index baed68989..000000000 --- a/repoman/pym/repoman/modules/scan/module.py +++ /dev/null @@ -1,102 +0,0 @@ - -''' -moudules/scan/module.py -Module loading and run list generator -''' - -import logging -import os -import yaml - -import portage -from portage.module import InvalidModuleName, Modules -from portage.util import stack_lists -from repoman import _not_installed -from repoman.config import ConfigError - -MODULES_PATH = os.path.dirname(__file__) -# initial development debug info -logging.debug("module path: %s", MODULES_PATH) - - -class ModuleConfig(object): - '''Holds the scan modules configuration information and - creates the ordered list of modulles to run''' - - def __init__(self, configpaths, valid_versions=None, repository_modules=False): - '''Module init - - @param configpaths: ordered list of filepaths to load - ''' - if repository_modules: - self.configpaths = [os.path.join(path, 'repository.yaml') for path in configpaths] - elif _not_installed: - self.configpaths = [os.path.realpath(os.path.join(os.path.dirname( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname( - os.path.dirname(__file__)))))), 'repoman/cnf/repository/repository.yaml'))] - else: - self.configpaths = [os.path.join(portage.const.EPREFIX or '/', - 'usr/share/repoman/repository/repository.yaml')] - logging.debug("ModuleConfig; configpaths: %s", self.configpaths) - - self.controller = Modules(path=MODULES_PATH, namepath="repoman.modules.scan") - logging.debug("ModuleConfig; module_names: %s", self.controller.module_names) - - self._configs = None - self.enabled = [] - self.pkgs_loop = [] - self.ebuilds_loop = [] - self.final_loop = [] - self.modules_forced = ['ebuild', 'mtime'] - self.load_configs(valid_versions=valid_versions) - for loop in ['pkgs', 'ebuilds', 'final']: - logging.debug("ModuleConfig; Processing loop %s", loop) - setattr(self, '%s_loop' % loop, self._determine_list(loop)) - self.linechecks = stack_lists(c['linechecks_modules'].split() for c in self._configs) - - def load_configs(self, configpaths=None, valid_versions=None): - '''load the config files in order - - @param configpaths: ordered list of filepaths to load - ''' - if configpaths: - self.configpaths = configpaths - elif not self.configpaths: - logging.error("ModuleConfig; Error: No repository.yaml files defined") - configs = [] - for path in self.configpaths: - logging.debug("ModuleConfig; Processing: %s", path) - if os.path.exists(path): - try: - with open(path, 'r') as inputfile: - configs.append(yaml.safe_load(inputfile)) - except IOError as error: - logging,error("Failed to load file: %s", inputfile) - logging.exception(error) - else: - if configs[-1]['version'] not in valid_versions: - raise ConfigError("Invalid file version: %s in: %s\nPlease upgrade repoman" % (configs['version'], path)) - logging.debug("ModuleConfig; completed : %s", path) - logging.debug("ModuleConfig; new _configs: %s", configs) - self._configs = configs - - def _determine_list(self, loop): - '''Determine the ordered list from the config data and - the moule_runsIn value in the module_spec - - @returns: list of modules - ''' - lists = [c['scan_modules'].split() for c in self._configs] - stacked = self.modules_forced + stack_lists(lists) - mlist = [] - try: - for mod in stacked: - logging.debug("ModuleConfig; checking loop %s, module: %s, in: %s", - loop, mod, self.controller.get_spec(mod, 'module_runsIn')) - if loop in self.controller.get_spec(mod, 'module_runsIn'): - mlist.append(mod) - except InvalidModuleName: - logging.error("ModuleConfig; unknown module: %s, skipping", mod) - - logging.debug("ModuleConfig; mlist: %s", mlist) - return mlist diff --git a/repoman/pym/repoman/modules/scan/options/__init__.py b/repoman/pym/repoman/modules/scan/options/__init__.py deleted file mode 100644 index ffce474e5..000000000 --- a/repoman/pym/repoman/modules/scan/options/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2015-2016 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Options plug-in module for repoman. -Performs option related actions on ebuilds.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'options', - 'description': doc, - 'provides':{ - 'options-module': { - 'name': "options", - 'sourcefile': "options", - 'class': "Options", - 'description': doc, - 'functions': ['is_forced'], - 'func_desc': { - }, - 'mod_kwargs': ['options', - ], - 'func_kwargs': { - }, - 'module_runsIn': ['ebuilds'], - }, - }, - 'version': 1, -} - diff --git a/repoman/pym/repoman/modules/scan/options/options.py b/repoman/pym/repoman/modules/scan/options/options.py deleted file mode 100644 index 443f01bd8..000000000 --- a/repoman/pym/repoman/modules/scan/options/options.py +++ /dev/null @@ -1,29 +0,0 @@ - -from repoman.modules.scan.scanbase import ScanBase - - -class Options(ScanBase): - - def __init__(self, **kwargs): - '''Class init function - - @param options: argparse options instance - ''' - self.options = kwargs.get('options') - - def is_forced(self, **kwargs): - '''Simple boolean function to trigger a skip past some additional checks - - @returns: dictionary - ''' - if self.options.force: - # The dep_check() calls are the most expensive QA test. If --force - # is enabled, there's no point in wasting time on these since the - # user is intent on forcing the commit anyway. - return True - return False - - @property - def runInEbuilds(self): - '''Ebuild level scans''' - return (True, [self.is_forced]) diff --git a/repoman/pym/repoman/modules/scan/scan.py b/repoman/pym/repoman/modules/scan/scan.py deleted file mode 100644 index d2a5f515b..000000000 --- a/repoman/pym/repoman/modules/scan/scan.py +++ /dev/null @@ -1,66 +0,0 @@ -# -*- coding:utf-8 -*- - -''' -moudules/scan.py -Module specific package scan list generator -''' - -import logging -import os -import sys - -from repoman.errors import caterror - - -def scan(repolevel, reposplit, startdir, categories, repo_settings): - '''Generate a list of pkgs to scan - - @param repolevel: integer, number of subdirectories deep from the tree root - @param reposplit: list of the path subdirs - @param startdir: the top level directory to begin scanning from - @param categories: list of known categories - @param repo_settings: repository settings instance - @returns: scanlist, sorted list of pkgs to scan - ''' - scanlist = [] - if repolevel == 2: - # we are inside a category directory - catdir = reposplit[-1] - if catdir not in categories: - caterror(catdir, repo_settings.repodir) - mydirlist = os.listdir(startdir) - for x in mydirlist: - if x == "CVS" or x.startswith("."): - continue - if os.path.isdir(startdir + "/" + x): - scanlist.append(catdir + "/" + x) - # repo_subdir = catdir + os.sep - elif repolevel == 1: - for x in categories: - if not os.path.isdir(startdir + "/" + x): - continue - for y in os.listdir(startdir + "/" + x): - if y == "CVS" or y.startswith("."): - continue - if os.path.isdir(startdir + "/" + x + "/" + y): - scanlist.append(x + "/" + y) - # repo_subdir = "" - elif repolevel == 3: - catdir = reposplit[-2] - if catdir not in categories: - caterror(catdir, repo_settings.repodir) - scanlist.append(catdir + "/" + reposplit[-1]) - # repo_subdir = scanlist[-1] + os.sep - else: - msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \ - ' from the current working directory' - logging.critical(msg) - sys.exit(1) - - # repo_subdir_len = len(repo_subdir) - scanlist.sort() - - logging.debug( - "Found the following packages to scan:\n%s" % '\n'.join(scanlist)) - - return scanlist diff --git a/repoman/pym/repoman/modules/scan/scanbase.py b/repoman/pym/repoman/modules/scan/scanbase.py deleted file mode 100644 index aea1bb121..000000000 --- a/repoman/pym/repoman/modules/scan/scanbase.py +++ /dev/null @@ -1,79 +0,0 @@ -# -*- coding:utf-8 -*- - - -class ScanBase(object): - '''Skeleton class for performing a scan for one or more items - to check in a pkg directory or ebuild.''' - - def __init__(self, **kwargs): - '''Class init - - @param kwargs: an optional dictionary of common repository - wide parameters that may be required. - ''' - # Since no two checks are identicle as to what kwargs are needed, - # this does not define any from it here. - super(ScanBase, self).__init__() - - """ # sample check - def check_foo(self, **kwargs): - '''Class check skeleton function. Define this for a - specific check to perform. - - @param kwargs: an optional dictionary of dynamic package and or ebuild - specific data that may be required. Dynamic data can - vary depending what checks have run before it. - So execution order can be important. - ''' - # Insert the code for the check here - # It should return a dictionary of at least {'continue': False} - # The continue attribute will default to False if not returned. - # This will allow the loop to continue with the next check in the list. - # Include any additional dynamic data that needs to be added or updated. - return False # used as a continue True/False value - """ - - @property - def runInPkgs(self): - '''Package level scans''' - # default no run (False) and empty list of functions to run - # override this method to define a function or - # functions to run in this process loop - # return a tuple of a boolean or boolean result and an ordered list - # of functions to run. ie: return (True, [self.check_foo]) - # in this way, it can be dynamically determined at run time, if - # later stage scans are to be run. - # This class instance is maintaned for all stages, so data can be - # carried over from stage to stage - # next stage is runInEbuilds - return (False, []) - - @property - def runInEbuilds(self): - '''Ebuild level scans''' - # default empty list of functions to run - # override this method to define a function or - # functions to run in this process loop - # return a tuple of a boolean or boolean result and an ordered list - # of functions to run. ie: return (True, [self.check_bar]) - # in this way, it can be dynamically determined at run time, if - # later stage scans are to be run. - # This class instance is maintaned for all stages, so data can be - # carried over from stage to stage - # next stage is runInFinal - return (False, []) - - @property - def runInFinal(self): - '''Final scans at the package level''' - # default empty list of functions to run - # override this method to define a function or - # functions to run in this process loop - # return a tuple of a boolean or boolean result and an ordered list - # of functions to run. ie: return (True, [self.check_baz]) - # in this way, it can be dynamically determined at run time, if - # later stage scans are to be run. - # This class instance is maintaned for all stages, so data can be - # carried over from stage to stage - # runInFinal is currently the last stage of scans performed. - return (False, []) diff --git a/repoman/pym/repoman/modules/vcs/None/__init__.py b/repoman/pym/repoman/modules/vcs/None/__init__.py deleted file mode 100644 index 285932541..000000000 --- a/repoman/pym/repoman/modules/vcs/None/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2014-2015 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """None (non vcs type) plug-in module for portage. -Performs various git actions and checks on repositories.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'None', - 'description': doc, - 'provides':{ - 'None-module': { - 'name': "None_status", - 'sourcefile': "status", - 'class': "Status", - 'description': doc, - 'functions': ['check', 'supports_gpg_sign', 'detect_conflicts'], - 'func_desc': { - }, - 'vcs_preserves_mtime': False, - 'needs_keyword_expansion': False, - }, - 'None-changes': { - 'name': "None_changes", - 'sourcefile': "changes", - 'class': "Changes", - 'description': doc, - 'functions': ['scan'], - 'func_desc': { - }, - }, - } -} diff --git a/repoman/pym/repoman/modules/vcs/None/changes.py b/repoman/pym/repoman/modules/vcs/None/changes.py deleted file mode 100644 index 46c38e257..000000000 --- a/repoman/pym/repoman/modules/vcs/None/changes.py +++ /dev/null @@ -1,50 +0,0 @@ -''' -None module Changes class submodule -''' - -from repoman.modules.vcs.changes import ChangesBase - - -class Changes(ChangesBase): - '''Class object to scan and hold the resultant data - for all changes to process. - ''' - - vcs = 'None' - - def __init__(self, options, repo_settings): - '''Class init - - @param options: the run time cli options - @param repo_settings: RepoSettings instance - ''' - super(Changes, self).__init__(options, repo_settings) - - def scan(self): - '''VCS type scan function, looks for all detectable changes''' - pass - - def add_items(self, autoadd): - '''Add files to the vcs's modified or new index - - @param autoadd: the files to add to the vcs modified index''' - pass - - def commit(self, myfiles, commitmessagefile): - '''None commit function - - @param commitfiles: list of files to commit - @param commitmessagefile: file containing the commit message - @returns: The sub-command exit value or 0 - ''' - commit_cmd = [] - # substitute a bogus vcs value for pretend output - commit_cmd.append("pretend") - commit_cmd.extend(self.vcs_settings.vcs_global_opts) - commit_cmd.append("commit") - commit_cmd.extend(self.vcs_settings.vcs_local_opts) - commit_cmd.extend(["-F", commitmessagefile]) - commit_cmd.extend(f.lstrip("./") for f in myfiles) - - print("(%s)" % (" ".join(commit_cmd),)) - return 0 diff --git a/repoman/pym/repoman/modules/vcs/None/status.py b/repoman/pym/repoman/modules/vcs/None/status.py deleted file mode 100644 index d6e5ca0e4..000000000 --- a/repoman/pym/repoman/modules/vcs/None/status.py +++ /dev/null @@ -1,53 +0,0 @@ -''' -None (non-VCS) module Status class submodule -''' - - -class Status(object): - '''Performs status checks on the svn repository''' - - def __init__(self, qatracker, eadded): - '''Class init - - @param qatracker: QATracker class instance - @param eadded: list - ''' - self.qatracker = qatracker - self.eadded = eadded - - def check(self, checkdir, checkdir_relative, xpkg): - '''Perform the svn status check - - @param checkdir: string of the directory being checked - @param checkdir_relative: string of the relative directory being checked - @param xpkg: string of the package being checked - @returns: boolean - ''' - return True - - @staticmethod - def detect_conflicts(options): - '''Are there any merge conflicts present in the VCS tracking system - - @param options: command line options - @returns: Boolean - ''' - return False - - @staticmethod - def supports_gpg_sign(): - '''Does this vcs system support gpg commit signatures - - @returns: Boolean - ''' - return False - - @staticmethod - def isVcsDir(dirname): - '''Is the directory belong to the vcs system - - @param dirname: string, directory name - @returns: Boolean - ''' - return False - diff --git a/repoman/pym/repoman/modules/vcs/__init__.py b/repoman/pym/repoman/modules/vcs/__init__.py deleted file mode 100644 index 84e837408..000000000 --- a/repoman/pym/repoman/modules/vcs/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ - -import os -from portage.module import Modules - -path = os.path.dirname(__file__) -# initial development debug info -#print("module path:", path) - -module_controller = Modules(path=path, namepath="repoman.modules.vcs") - -# initial development debug info -#print(module_controller.module_names) -module_names = module_controller.module_names[:] - diff --git a/repoman/pym/repoman/modules/vcs/bzr/__init__.py b/repoman/pym/repoman/modules/vcs/bzr/__init__.py deleted file mode 100644 index 4490ed86c..000000000 --- a/repoman/pym/repoman/modules/vcs/bzr/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2014-2015 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Bazaar (bzr) plug-in module for portage. -Performs variaous Bazaar actions and checks on repositories.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'bzr', - 'description': doc, - 'provides':{ - 'bzr-module': { - 'name': "bzr_status", - 'sourcefile': "status", - 'class': "Status", - 'description': doc, - 'functions': ['check', 'supports_gpg_sign', 'detect_conflicts'], - 'func_desc': { - }, - 'vcs_preserves_mtime': True, - 'needs_keyword_expansion': False, - }, - 'bzr-changes': { - 'name': "bzr_changes", - 'sourcefile': "changes", - 'class': "Changes", - 'description': doc, - 'functions': ['scan'], - 'func_desc': { - }, - }, - } -} diff --git a/repoman/pym/repoman/modules/vcs/bzr/changes.py b/repoman/pym/repoman/modules/vcs/bzr/changes.py deleted file mode 100644 index 4d4808c08..000000000 --- a/repoman/pym/repoman/modules/vcs/bzr/changes.py +++ /dev/null @@ -1,68 +0,0 @@ -''' -Bazaar module Changes class submodule -''' - -from repoman.modules.vcs.changes import ChangesBase -from repoman._subprocess import repoman_popen -from repoman._portage import portage -from portage import os -from portage.package.ebuild.digestgen import digestgen - -class Changes(ChangesBase): - '''Class object to scan and hold the resultant data - for all changes to process. - ''' - - vcs = 'bzr' - - def __init__(self, options, repo_settings): - '''Class init - - @param options: the run time cli options - @param repo_settings: RepoSettings instance - ''' - super(Changes, self).__init__(options, repo_settings) - - def _scan(self): - '''VCS type scan function, looks for all detectable changes''' - with repoman_popen("bzr status -S .") as f: - bzrstatus = f.readlines() - self.changed = [ - "./" + elem.split()[-1:][0].split('/')[-1:][0] - for elem in bzrstatus - if elem and elem[1:2] == "M"] - self.new = [ - "./" + elem.split()[-1:][0].split('/')[-1:][0] - for elem in bzrstatus - if elem and (elem[1:2] == "NK" or elem[0:1] == "R")] - self.removed = [ - "./" + elem.split()[-3:-2][0].split('/')[-1:][0] - for elem in bzrstatus - if elem and (elem[1:2] == "K" or elem[0:1] == "R")] - self.bzrstatus = bzrstatus - # Bazaar expands nothing. - - @property - def unadded(self): - '''Bazzar method of getting the unadded files in the repository''' - if self._unadded is not None: - return self._unadded - self._unadded = [ - "./" + elem.rstrip().split()[1].split('/')[-1:][0] - for elem in self.bzrstatus - if elem.startswith("?") or elem[0:2] == " D"] - return self._unadded - - def digest_regen(self, updates, removed, manifests, scanner, broken_changelog_manifests): - '''Regenerate manifests - - @param updates: updated files - @param removed: removed files - @param manifests: Manifest files - @param scanner: The repoman.scanner.Scanner instance - @param broken_changelog_manifests: broken changelog manifests - ''' - if broken_changelog_manifests: - for x in broken_changelog_manifests: - self.repoman_settings["O"] = os.path.join(self.repo_settings.repodir, x) - digestgen(mysettings=self.repoman_settings, myportdb=self.repo_settings.portdb) diff --git a/repoman/pym/repoman/modules/vcs/bzr/status.py b/repoman/pym/repoman/modules/vcs/bzr/status.py deleted file mode 100644 index 199e7f399..000000000 --- a/repoman/pym/repoman/modules/vcs/bzr/status.py +++ /dev/null @@ -1,70 +0,0 @@ -''' -Bazaar module Status class submodule -''' - -from repoman._portage import portage -from portage import os -from repoman._subprocess import repoman_popen - - -class Status(object): - '''Performs status checks on the svn repository''' - - def __init__(self, qatracker, eadded): - '''Class init - - @param qatracker: QATracker class instance - @param eadded: list - ''' - self.qatracker = qatracker - self.eadded = eadded - - def check(self, checkdir, checkdir_relative, xpkg): - '''Perform the svn status check - - @param checkdir: string of the directory being checked - @param checkdir_relative: string of the relative directory being checked - @param xpkg: string of the package being checked - @returns: boolean - ''' - try: - myf = repoman_popen( - "bzr ls -v --kind=file " + - portage._shell_quote(checkdir)) - myl = myf.readlines() - myf.close() - except IOError: - raise - for l in myl: - if l[1:2] == "?": - continue - l = l.split()[-1] - if l[-7:] == ".ebuild": - self.eadded.append(os.path.basename(l[:-7])) - return True - - @staticmethod - def detect_conflicts(options): - '''Are there any merge conflicts present in the VCS tracking system - - @param options: command line options - @returns: Boolean - ''' - return False - - @staticmethod - def supports_gpg_sign(): - '''Does this vcs system support gpg commit signatures - - @returns: Boolean - ''' - return False - - @staticmethod - def isVcsDir(dirname): - '''Does the directory belong to the vcs system - - @param dirname: string, directory name - @returns: Boolean - ''' - return dirname in [".bzr"] diff --git a/repoman/pym/repoman/modules/vcs/changes.py b/repoman/pym/repoman/modules/vcs/changes.py deleted file mode 100644 index aa4923f8f..000000000 --- a/repoman/pym/repoman/modules/vcs/changes.py +++ /dev/null @@ -1,169 +0,0 @@ -''' -Base Changes class -''' - -import logging -import os -import subprocess -import sys -from itertools import chain - -from repoman._portage import portage -from portage import _unicode_encode -from portage.process import spawn - - -class ChangesBase(object): - '''Base Class object to scan and hold the resultant data - for all changes to process. - ''' - - vcs = 'None' - - def __init__(self, options, repo_settings): - '''Class init function - - @param options: the run time cli options - @param repo_settings: RepoSettings instance - ''' - self.options = options - self.repo_settings = repo_settings - self.repoman_settings = repo_settings.repoman_settings - self.vcs_settings = repo_settings.vcs_settings - self._reset() - - def _reset(self): - '''Reset the class variables for a new run''' - self.new_ebuilds = set() - self.ebuilds = set() - self.changelogs = set() - self.changed = [] - self.new = [] - self.removed = [] - self.no_expansion = set() - self._expansion = None - self._deleted = None - self._unadded = None - - def scan(self): - '''Scan the vcs for detectable changes. - - base method which calls the subclassing VCS module's _scan() - then updates some classwide variables. - ''' - self._reset() - - if self.vcs: - self._scan() - self.new_ebuilds.update(x for x in self.new if x.endswith(".ebuild")) - self.ebuilds.update(x for x in self.changed if x.endswith(".ebuild")) - self.changelogs.update( - x for x in chain(self.changed, self.new) - if os.path.basename(x) == "ChangeLog") - - def _scan(self): - '''Placeholder for subclassing''' - pass - - @property - def has_deleted(self): - '''Placeholder for VCS that requires manual deletion of files''' - return self.deleted != [] - - @property - def has_changes(self): - '''Placeholder for VCS repo common has changes result''' - changed = self.changed or self.new or self.removed or self.deleted - return changed != [] - - @property - def unadded(self): - '''Override this function as needed''' - return [] - - @property - def deleted(self): - '''Override this function as needed''' - return [] - - @property - def expansion(self): - '''Override this function as needed''' - return {} - - def thick_manifest(self, updates, headers, no_expansion, expansion): - '''Create a thick manifest - - @param updates: - @param headers: - @param no_expansion: - @param expansion: - ''' - pass - - def digest_regen(self, updates, removed, manifests, scanner, - broken_changelog_manifests): - '''Regenerate manifests - - @param updates: updated files - @param removed: removed files - @param manifests: Manifest files - @param scanner: The repoman.scanner.Scanner instance - @param broken_changelog_manifests: broken changelog manifests - ''' - pass - - @staticmethod - def clear_attic(headers): - '''Old CVS leftover - - @param headers: file headers''' - pass - - def update_index(self, mymanifests, myupdates): - '''Update the vcs's modified index if it is needed - - @param mymanifests: manifest files updated - @param myupdates: other files updated''' - pass - - def add_items(self, autoadd): - '''Add files to the vcs's modified or new index - - @param autoadd: the files to add to the vcs modified index''' - add_cmd = [self.vcs, "add"] - add_cmd += autoadd - if self.options.pretend: - portage.writemsg_stdout( - "(%s)\n" % " ".join(add_cmd), - noiselevel=-1) - else: - add_cmd = [_unicode_encode(arg) for arg in add_cmd] - retcode = subprocess.call(add_cmd) - if retcode != os.EX_OK: - logging.error( - "Exiting on %s error code: %s\n", self.vcs_settings.vcs, retcode) - sys.exit(retcode) - - - def commit(self, commitfiles, commitmessagefile): - '''Common generic commit function - - @param commitfiles: list of files to commit - @param commitmessagefile: file containing the commit message - @returns: The sub-command exit value or 0 - ''' - commit_cmd = [] - commit_cmd.append(self.vcs) - commit_cmd.extend(self.vcs_settings.vcs_global_opts) - commit_cmd.append("commit") - commit_cmd.extend(self.vcs_settings.vcs_local_opts) - commit_cmd.extend(["-F", commitmessagefile]) - commit_cmd.extend(f.lstrip("./") for f in commitfiles) - - if self.options.pretend: - print("(%s)" % (" ".join(commit_cmd),)) - return 0 - else: - retval = spawn(commit_cmd, env=self.repo_settings.commit_env) - return retval diff --git a/repoman/pym/repoman/modules/vcs/cvs/__init__.py b/repoman/pym/repoman/modules/vcs/cvs/__init__.py deleted file mode 100644 index 0b4587bc6..000000000 --- a/repoman/pym/repoman/modules/vcs/cvs/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2014-2015 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """CVS (cvs) plug-in module for portage. -Performs variaous CVS actions and checks on repositories.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'cvs', - 'description': doc, - 'provides':{ - 'cvs-status': { - 'name': "cvs_status", - 'sourcefile': "status", - 'class': "Status", - 'description': doc, - 'functions': ['check', 'supports_gpg_sign', 'detect_conflicts'], - 'func_desc': { - }, - 'vcs_preserves_mtime': True, - 'needs_keyword_expansion': True, - }, - 'cvs-changes': { - 'name': "cvs_changes", - 'sourcefile': "changes", - 'class': "Changes", - 'description': doc, - 'functions': ['scan'], - 'func_desc': { - }, - }, - } -} diff --git a/repoman/pym/repoman/modules/vcs/cvs/changes.py b/repoman/pym/repoman/modules/vcs/cvs/changes.py deleted file mode 100644 index 583a96f7c..000000000 --- a/repoman/pym/repoman/modules/vcs/cvs/changes.py +++ /dev/null @@ -1,118 +0,0 @@ -''' -CVS module Changes class submodule -''' - -import re -from itertools import chain - -from repoman._portage import portage -from repoman.modules.vcs.changes import ChangesBase -from repoman.modules.vcs.vcs import vcs_files_to_cps -from repoman._subprocess import repoman_getstatusoutput - -from portage import _encodings, _unicode_encode -from portage import cvstree, os -from portage.output import green -from portage.package.ebuild.digestgen import digestgen - - -class Changes(ChangesBase): - '''Class object to scan and hold the resultant data - for all changes to process. - ''' - - vcs = 'cvs' - - def __init__(self, options, repo_settings): - '''Class init - - @param options: the run time cli options - @param repo_settings: RepoSettings instance - ''' - super(Changes, self).__init__(options, repo_settings) - self._tree = None - - def _scan(self): - '''VCS type scan function, looks for all detectable changes''' - self._tree = portage.cvstree.getentries("./", recursive=1) - self.changed = cvstree.findchanged(self._tree, recursive=1, basedir="./") - self.new = cvstree.findnew(self._tree, recursive=1, basedir="./") - self.removed = cvstree.findremoved(self._tree, recursive=1, basedir="./") - bin_blob_pattern = re.compile("^-kb$") - self.no_expansion = set(portage.cvstree.findoption( - self._tree, bin_blob_pattern, recursive=1, basedir="./")) - - @property - def unadded(self): - '''VCS method of getting the unadded files in the repository''' - if self._unadded is not None: - return self._unadded - self._unadded = portage.cvstree.findunadded(self._tree, recursive=1, basedir="./") - return self._unadded - - @staticmethod - def clear_attic(headers): - '''Clear the attic (inactive files) - - @param headers: file headers - ''' - cvs_header_re = re.compile(br'^#\s*\$Header.*\$$') - attic_str = b'/Attic/' - attic_replace = b'/' - for x in headers: - f = open( - _unicode_encode(x, encoding=_encodings['fs'], errors='strict'), - mode='rb') - mylines = f.readlines() - f.close() - modified = False - for i, line in enumerate(mylines): - if cvs_header_re.match(line) is not None and \ - attic_str in line: - mylines[i] = line.replace(attic_str, attic_replace) - modified = True - if modified: - portage.util.write_atomic(x, b''.join(mylines), mode='wb') - - def thick_manifest(self, updates, headers, no_expansion, expansion): - '''Create a thick manifest - - @param updates: - @param headers: - @param no_expansion: - @param expansion: - ''' - headerstring = r"'\$(Header|Id).*\$'" - - for _file in updates: - - # for CVS, no_expansion contains files that are excluded from expansion - if _file in no_expansion: - continue - - _out = repoman_getstatusoutput( - "egrep -q %s %s" % (headerstring, portage._shell_quote(_file))) - if _out[0] == 0: - headers.append(_file) - - print("%s have headers that will change." % green(str(len(headers)))) - print( - "* Files with headers will" - " cause the manifests to be changed and committed separately.") - - def digest_regen(self, updates, removed, manifests, scanner, broken_changelog_manifests): - '''Regenerate manifests - - @param updates: updated files - @param removed: removed files - @param manifests: Manifest files - @param scanner: The repoman.scanner.Scanner instance - @param broken_changelog_manifests: broken changelog manifests - ''' - if updates or removed: - for x in sorted(vcs_files_to_cps( - chain(updates, removed, manifests), - self.repo_settings.repodir, - scanner.repolevel, scanner.reposplit, scanner.categories)): - self.repoman_settings["O"] = os.path.join(self.repo_settings.repodir, x) - digestgen(mysettings=self.repoman_settings, myportdb=self.repo_settings.portdb) diff --git a/repoman/pym/repoman/modules/vcs/cvs/status.py b/repoman/pym/repoman/modules/vcs/cvs/status.py deleted file mode 100644 index 3ec88f125..000000000 --- a/repoman/pym/repoman/modules/vcs/cvs/status.py +++ /dev/null @@ -1,131 +0,0 @@ -''' -CVS module Status class submodule -''' - -import logging -import subprocess -import sys - -from repoman._portage import portage -from portage import os -from portage.const import BASH_BINARY -from portage.output import red, green -from portage import _unicode_encode, _unicode_decode - - -class Status(object): - '''Performs status checks on the svn repository''' - - def __init__(self, qatracker, eadded): - '''Class init - - @param qatracker: QATracker class instance - @param eadded: list - ''' - self.qatracker = qatracker - self.eadded = eadded - - def check(self, checkdir, checkdir_relative, xpkg): - '''Perform the svn status check - - @param checkdir: string of the directory being checked - @param checkdir_relative: string of the relative directory being checked - @param xpkg: string of the package being checked - @returns: boolean - ''' - try: - myf = open(checkdir + "/CVS/Entries", "r") - myl = myf.readlines() - myf.close() - except IOError: - self.qatracker.add_error( - "CVS/Entries.IO_error", checkdir + "/CVS/Entries") - return True - for l in myl: - if l[0] != "/": - continue - splitl = l[1:].split("/") - if not len(splitl): - continue - if splitl[0][-7:] == ".ebuild": - self.eadded.append(splitl[0][:-7]) - return True - - @staticmethod - def detect_conflicts(options): - """Determine if the checkout has cvs conflicts. - - TODO(antarus): Also this should probably not call sys.exit() as - repoman is run on >1 packages and one failure should not cause - subsequent packages to fail. - - Returns: - None (calls sys.exit on fatal problems) - """ - - cmd = (r"cvs -n up 2>/dev/null | " - r"egrep '^[^\?] .*' | " - r"egrep -v '^. .*/digest-[^/]+|^cvs server: .* -- ignored$'") - msg = ("Performing a %s with a little magic grep to check for updates." - % green("cvs -n up")) - - logging.info(msg) - # Use Popen instead of getstatusoutput(), in order to avoid - # unicode handling problems (see bug #310789). - args = [BASH_BINARY, "-c", cmd] - args = [_unicode_encode(x) for x in args] - proc = subprocess.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - out = _unicode_decode(proc.communicate()[0]) - proc.wait() - mylines = out.splitlines() - myupdates = [] - for line in mylines: - if not line: - continue - - # [ ] Unmodified (SVN) [U] Updates [P] Patches - # [M] Modified [A] Added [R] Removed / Replaced - # [D] Deleted - if line[0] not in " UPMARD": - # Stray Manifest is fine, we will readd it anyway. - if line[0] == '?' and line[1:].lstrip() == 'Manifest': - continue - logging.error(red( - "!!! Please fix the following issues reported " - "from cvs: %s" % green("(U,P,M,A,R,D are ok)"))) - logging.error(red( - "!!! Note: This is a pretend/no-modify pass...")) - logging.error(out) - sys.exit(1) - elif line[0] in "UP": - myupdates.append(line[2:]) - - if myupdates: - logging.info(green("Fetching trivial updates...")) - if options.pretend: - logging.info("(cvs update " + " ".join(myupdates) + ")") - retval = os.EX_OK - else: - retval = os.system("cvs update " + " ".join(myupdates)) - if retval != os.EX_OK: - logging.fatal("!!! cvs exited with an error. Terminating.") - sys.exit(retval) - return False - - @staticmethod - def supports_gpg_sign(): - '''Does this vcs system support gpg commit signatures - - @returns: Boolean - ''' - return False - - @staticmethod - def isVcsDir(dirname): - '''Does the directory belong to the vcs system - - @param dirname: string, directory name - @returns: Boolean - ''' - return dirname in ["CVS"] diff --git a/repoman/pym/repoman/modules/vcs/git/__init__.py b/repoman/pym/repoman/modules/vcs/git/__init__.py deleted file mode 100644 index eecd4a1d0..000000000 --- a/repoman/pym/repoman/modules/vcs/git/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2014-2015 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Git (git) plug-in module for portage. -Performs variaous git actions and checks on repositories.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'git', - 'description': doc, - 'provides':{ - 'git-module': { - 'name': "git_status", - 'sourcefile': "status", - 'class': "Status", - 'description': doc, - 'functions': ['check', 'supports_gpg_sign', 'detect_conflicts'], - 'func_desc': { - }, - 'vcs_preserves_mtime': False, - 'needs_keyword_expansion': False, - }, - 'git-changes': { - 'name': "git_changes", - 'sourcefile': "changes", - 'class': "Changes", - 'description': doc, - 'functions': ['scan'], - 'func_desc': { - }, - }, - } -} diff --git a/repoman/pym/repoman/modules/vcs/git/changes.py b/repoman/pym/repoman/modules/vcs/git/changes.py deleted file mode 100644 index 7e9ac1eb5..000000000 --- a/repoman/pym/repoman/modules/vcs/git/changes.py +++ /dev/null @@ -1,120 +0,0 @@ -''' -Git module Changes class submodule -''' - -import logging -import sys - -from repoman.modules.vcs.changes import ChangesBase -from repoman._subprocess import repoman_popen -from repoman._portage import portage -from portage import os -from portage.package.ebuild.digestgen import digestgen -from portage.process import spawn -from portage.util import writemsg_level - - -class Changes(ChangesBase): - '''Class object to scan and hold the resultant data - for all changes to process. - ''' - - vcs = 'git' - - def __init__(self, options, repo_settings): - '''Class init - - @param options: the run time cli options - @param repo_settings: RepoSettings instance - ''' - super(Changes, self).__init__(options, repo_settings) - - def _scan(self): - '''VCS type scan function, looks for all detectable changes''' - with repoman_popen( - "git diff-index --name-only " - "--relative --diff-filter=M HEAD") as f: - changed = f.readlines() - self.changed = ["./" + elem[:-1] for elem in changed] - del changed - - with repoman_popen( - "git diff-index --name-only " - "--relative --diff-filter=A HEAD") as f: - new = f.readlines() - self.new = ["./" + elem[:-1] for elem in new] - del new - - with repoman_popen( - "git diff-index --name-only " - "--relative --diff-filter=D HEAD") as f: - removed = f.readlines() - self.removed = ["./" + elem[:-1] for elem in removed] - del removed - - @property - def unadded(self): - '''VCS method of getting the unadded files in the repository''' - if self._unadded is not None: - return self._unadded - # get list of files not under version control or missing - with repoman_popen("git ls-files --others") as f: - unadded = f.readlines() - self._unadded = ["./" + elem[:-1] for elem in unadded] - del unadded - return self._unadded - - def digest_regen(self, updates, removed, manifests, scanner, broken_changelog_manifests): - '''Regenerate manifests - - @param updates: updated files - @param removed: removed files - @param manifests: Manifest files - @param scanner: The repoman.scanner.Scanner instance - @param broken_changelog_manifests: broken changelog manifests - ''' - if broken_changelog_manifests: - for x in broken_changelog_manifests: - self.repoman_settings["O"] = os.path.join(self.repo_settings.repodir, x) - digestgen(mysettings=self.repoman_settings, myportdb=self.repo_settings.portdb) - - def update_index(self, mymanifests, myupdates): - '''Update the vcs's modified index if it is needed - - @param mymanifests: manifest files updated - @param myupdates: other files updated''' - # It's not safe to use the git commit -a option since there might - # be some modified files elsewhere in the working tree that the - # user doesn't want to commit. Therefore, call git update-index - # in order to ensure that the index is updated with the latest - # versions of all new and modified files in the relevant portion - # of the working tree. - myfiles = mymanifests + myupdates - myfiles.sort() - update_index_cmd = ["git", "update-index"] - update_index_cmd.extend(f.lstrip("./") for f in myfiles) - if self.options.pretend: - print("(%s)" % (" ".join(update_index_cmd),)) - else: - retval = spawn(update_index_cmd, env=os.environ) - if retval != os.EX_OK: - writemsg_level( - "!!! Exiting on %s (shell) " - "error code: %s\n" % (self.vcs_settings.vcs, retval), - level=logging.ERROR, noiselevel=-1) - sys.exit(retval) - - def commit(self, myfiles, commitmessagefile): - '''Git commit function - - @param commitfiles: list of files to commit - @param commitmessagefile: file containing the commit message - @returns: The sub-command exit value or 0 - ''' - retval = super(Changes, self).commit(myfiles, commitmessagefile) - if retval != os.EX_OK: - if self.repo_settings.repo_config.sign_commit and not self.vcs_settings.status.supports_gpg_sign(): - # Inform user that newer git is needed (bug #403323). - logging.error( - "Git >=1.7.9 is required for signed commits!") - return retval diff --git a/repoman/pym/repoman/modules/vcs/git/status.py b/repoman/pym/repoman/modules/vcs/git/status.py deleted file mode 100644 index e5aa9c741..000000000 --- a/repoman/pym/repoman/modules/vcs/git/status.py +++ /dev/null @@ -1,78 +0,0 @@ -''' -Git module Status class submodule -''' - -import re - -from repoman._portage import portage -from portage import os -from repoman._subprocess import repoman_popen, repoman_getstatusoutput - - -class Status(object): - '''Performs status checks on the git repository''' - - def __init__(self, qatracker, eadded): - '''Class init - - @param qatracker: QATracker class instance - @param eadded: list - ''' - self.qatracker = qatracker - self.eadded = eadded - - def check(self, checkdir, checkdir_relative, xpkg): - '''Perform the git status check - - @param checkdir: string of the directory being checked - @param checkdir_relative: string of the relative directory being checked - @param xpkg: string of the package being checked - @returns: boolean - ''' - with repoman_popen( - "git ls-files --others %s" % - (portage._shell_quote(checkdir_relative),)) as myf: - for l in myf: - if l[:-1][-7:] == ".ebuild": - self.qatracker.add_error( - "ebuild.notadded", - os.path.join(xpkg, os.path.basename(l[:-1]))) - return True - - @staticmethod - def detect_conflicts(options): - '''Are there any merge conflicts present in the VCS tracking system - - @param options: command line options - @returns: Boolean - ''' - return False - - @staticmethod - def supports_gpg_sign(): - '''Does this vcs system support gpg commit signatures - - @returns: Boolean - ''' - status, cmd_output = \ - repoman_getstatusoutput("git --version") - cmd_output = cmd_output.split() - if cmd_output: - version = re.match(r'^(\d+)\.(\d+)\.(\d+)', cmd_output[-1]) - if version is not None: - version = [int(x) for x in version.groups()] - if version[0] > 1 or \ - (version[0] == 1 and version[1] > 7) or \ - (version[0] == 1 and version[1] == 7 and version[2] >= 9): - return True - return False - - @staticmethod - def isVcsDir(dirname): - '''Does the directory belong to the vcs system - - @param dirname: string, directory name - @returns: Boolean - ''' - return dirname in [".git"] - diff --git a/repoman/pym/repoman/modules/vcs/hg/__init__.py b/repoman/pym/repoman/modules/vcs/hg/__init__.py deleted file mode 100644 index 2e39970f7..000000000 --- a/repoman/pym/repoman/modules/vcs/hg/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2014-2015 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Mercurial (hg) plug-in module for portage. -Performs variaous mercurial actions and checks on repositories.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'hg', - 'description': doc, - 'provides':{ - 'hg-module': { - 'name': "hg_status", - 'sourcefile': "status", - 'class': "Status", - 'description': doc, - 'functions': ['check', 'supports_gpg_sign', 'detect_conflicts'], - 'func_desc': { - }, - 'vcs_preserves_mtime': False, - 'needs_keyword_expansion': False, - }, - 'hg-changes': { - 'name': "hg_changes", - 'sourcefile': "changes", - 'class': "Changes", - 'description': doc, - 'functions': ['scan'], - 'func_desc': { - }, - }, - } -} diff --git a/repoman/pym/repoman/modules/vcs/hg/changes.py b/repoman/pym/repoman/modules/vcs/hg/changes.py deleted file mode 100644 index 867057545..000000000 --- a/repoman/pym/repoman/modules/vcs/hg/changes.py +++ /dev/null @@ -1,105 +0,0 @@ -''' -Mercurial module Changes class submodule -''' - -from repoman.modules.vcs.changes import ChangesBase -from repoman._subprocess import repoman_popen -from repoman._portage import portage -from portage import os -from portage.package.ebuild.digestgen import digestgen -from portage.process import spawn - - -class Changes(ChangesBase): - '''Class object to scan and hold the resultant data - for all changes to process. - ''' - - vcs = 'hg' - - def __init__(self, options, repo_settings): - '''Class init - - @param options: the run time cli options - @param repo_settings: RepoSettings instance - ''' - super(Changes, self).__init__(options, repo_settings) - - def _scan(self): - '''VCS type scan function, looks for all detectable changes''' - with repoman_popen("hg status --no-status --modified .") as f: - changed = f.readlines() - self.changed = ["./" + elem.rstrip() for elem in changed] - del changed - - with repoman_popen("hg status --no-status --added .") as f: - new = f.readlines() - self.new = ["./" + elem.rstrip() for elem in new] - del new - - with repoman_popen("hg status --no-status --removed .") as f: - removed = f.readlines() - self.removed = ["./" + elem.rstrip() for elem in removed] - del removed - - @property - def unadded(self): - '''VCS method of getting the unadded files in the repository''' - if self._unadded is not None: - return self._unadded - with repoman_popen("hg status --no-status --unknown .") as f: - unadded = f.readlines() - self._unadded = ["./" + elem.rstrip() for elem in unadded] - del unadded - return self._unadded - - @property - def deleted(self): - '''VCS method of getting the deleted files in the repository''' - if self._deleted is not None: - return self._deleted - # Mercurial doesn't handle manually deleted files as removed from - # the repository, so the user need to remove them before commit, - # using "hg remove [FILES]" - with repoman_popen("hg status --no-status --deleted .") as f: - deleted = f.readlines() - self._deleted = ["./" + elem.rstrip() for elem in deleted] - del deleted - return self._deleted - - - def digest_regen(self, updates, removed, manifests, scanner, broken_changelog_manifests): - '''Regenerate manifests - - @param updates: updated files - @param removed: removed files - @param manifests: Manifest files - @param scanner: The repoman.scanner.Scanner instance - @param broken_changelog_manifests: broken changelog manifests - ''' - if broken_changelog_manifests: - for x in broken_changelog_manifests: - self.repoman_settings["O"] = os.path.join(self.repo_settings.repodir, x) - digestgen(mysettings=self.repoman_settings, myportdb=self.repo_settings.portdb) - - def commit(self, myfiles, commitmessagefile): - '''Hg commit function - - @param commitfiles: list of files to commit - @param commitmessagefile: file containing the commit message - @returns: The sub-command exit value or 0 - ''' - commit_cmd = [] - commit_cmd.append(self.vcs) - commit_cmd.extend(self.vcs_settings.vcs_global_opts) - commit_cmd.append("commit") - commit_cmd.extend(self.vcs_settings.vcs_local_opts) - commit_cmd.extend(["--logfile", commitmessagefile]) - commit_cmd.extend(myfiles) - - if self.options.pretend: - print("(%s)" % (" ".join(commit_cmd),)) - return 0 - else: - retval = spawn(commit_cmd, env=self.repo_settings.commit_env) - return retval diff --git a/repoman/pym/repoman/modules/vcs/hg/status.py b/repoman/pym/repoman/modules/vcs/hg/status.py deleted file mode 100644 index 8443554f5..000000000 --- a/repoman/pym/repoman/modules/vcs/hg/status.py +++ /dev/null @@ -1,65 +0,0 @@ -''' -Mercurial module Status class submodule -''' - -from repoman._portage import portage -from portage import os -from repoman._subprocess import repoman_popen - - -class Status(object): - '''Performs status checks on the svn repository''' - - def __init__(self, qatracker, eadded): - '''Class init - - @param qatracker: QATracker class instance - @param eadded: list - ''' - self.qatracker = qatracker - self.eadded = eadded - - def check(self, checkdir, checkdir_relative, xpkg): - '''Perform the svn status check - - @param checkdir: string of the directory being checked - @param checkdir_relative: string of the relative directory being checked - @param xpkg: string of the package being checked - @returns: boolean - ''' - myf = repoman_popen( - "hg status --no-status --unknown %s" % - (portage._shell_quote(checkdir_relative),)) - for l in myf: - if l[:-1][-7:] == ".ebuild": - self.qatracker.add_error( - "ebuild.notadded", - os.path.join(xpkg, os.path.basename(l[:-1]))) - myf.close() - return True - - @staticmethod - def detect_conflicts(options): - '''Are there any merge conflicts present in the VCS tracking system - - @param options: command line options - @returns: Boolean - ''' - return False - - @staticmethod - def supports_gpg_sign(): - '''Does this vcs system support gpg commit signatures - - @returns: Boolean - ''' - return False - - @staticmethod - def isVcsDir(dirname): - '''Does the directory belong to the vcs system - - @param dirname: string, directory name - @returns: Boolean - ''' - return dirname in [".hg"] diff --git a/repoman/pym/repoman/modules/vcs/settings.py b/repoman/pym/repoman/modules/vcs/settings.py deleted file mode 100644 index a8e91dd27..000000000 --- a/repoman/pym/repoman/modules/vcs/settings.py +++ /dev/null @@ -1,108 +0,0 @@ -''' -Repoman VCSSettings modules -''' - -from __future__ import print_function, unicode_literals - -import logging -import sys - -from portage.output import red -from repoman.modules.vcs import module_controller, module_names -from repoman.modules.vcs.vcs import FindVCS -from repoman.qa_tracker import QATracker - - -class VCSSettings(object): - '''Holds various VCS settings''' - - def __init__(self, options=None, repoman_settings=None, repo_settings=None): - '''Class init function - - @param options: the run time cli options - @param repoman_settings: portage.config settings instance - @param repo_settings: RepoSettings instance - ''' - self.options = options - self.repoman_settings = repoman_settings - self.repo_settings = repo_settings - if options.vcs: - if options.vcs in module_names: - self.vcs = options.vcs - else: - self.vcs = None - else: - vcses = FindVCS() - if len(vcses) > 1: - print(red( - '*** Ambiguous workdir -- more than one VCS found' - ' at the same depth: %s.' % ', '.join(vcses))) - print(red( - '*** Please either clean up your workdir' - ' or specify --vcs option.')) - sys.exit(1) - elif vcses: - self.vcs = vcses[0] - else: - self.vcs = None - - if options.if_modified == "y" and self.vcs is None: - logging.info( - "Not in a version controlled repository; " - "disabling --if-modified.") - options.if_modified = "n" - - # initialize our instance placeholders - self._status = None - self._changes = None - # get our vcs plugin controller and available module names - self.module_controller = module_controller - self.module_names = module_names - - # Disable copyright/mtime check if vcs does not preserve mtime (bug #324075). - if str(self.vcs) in self.module_controller.parents: - self.vcs_preserves_mtime = module_controller.modules[ - "%s_status" % self.vcs]['vcs_preserves_mtime'] - else: - self.vcs_preserves_mtime = False - logging.error("VCSSettings: Unknown VCS type: %s", self.vcs) - logging.error("Available modules: %s", module_controller.parents) - - self.needs_keyword_expansion = module_controller.modules[ - "%s_status" % self.vcs]['needs_keyword_expansion'] - self.vcs_local_opts = repoman_settings.get( - "REPOMAN_VCS_LOCAL_OPTS", "").split() - self.vcs_global_opts = repoman_settings.get( - "REPOMAN_VCS_GLOBAL_OPTS") - if self.vcs_global_opts is None: - if self.vcs in ('cvs', 'svn'): - self.vcs_global_opts = "-q" - else: - self.vcs_global_opts = "" - self.vcs_global_opts = self.vcs_global_opts.split() - - if options.mode == 'commit' and not options.pretend and not self.vcs: - logging.info( - "Not in a version controlled repository; " - "enabling pretend mode.") - options.pretend = True - self.qatracker = QATracker() - self.eadded = [] - - @property - def status(self): - '''Initializes and returns the class instance - of the vcs's Status class''' - if not self._status: - status = self.module_controller.get_class('%s_status' % self.vcs) - self._status = status(self.qatracker, self.eadded) - return self._status - - @property - def changes(self): - '''Initializes and returns the class instance - of the vcs's Changes class''' - if not self._changes: - changes = self.module_controller.get_class('%s_changes' % self.vcs) - self._changes = changes(self.options, self.repo_settings) - return self._changes diff --git a/repoman/pym/repoman/modules/vcs/svn/__init__.py b/repoman/pym/repoman/modules/vcs/svn/__init__.py deleted file mode 100644 index 6bb0b9af4..000000000 --- a/repoman/pym/repoman/modules/vcs/svn/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2014-2015 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -doc = """Subversion (svn) plug-in module for portage. -Performs variaous subversion actions and checks on repositories.""" -__doc__ = doc[:] - - -module_spec = { - 'name': 'svn', - 'description': doc, - 'provides':{ - 'svn-module': { - 'name': "svn_status", - 'sourcefile': "status", - 'class': "Status", - 'description': doc, - 'functions': ['check', 'supports_gpg_sign', 'detect_conflicts'], - 'func_desc': { - }, - 'vcs_preserves_mtime': False, - 'needs_keyword_expansion': True, - }, - 'svn-changes': { - 'name': "svn_changes", - 'sourcefile': "changes", - 'class': "Changes", - 'description': doc, - 'functions': ['scan'], - 'func_desc': { - }, - }, - } -} diff --git a/repoman/pym/repoman/modules/vcs/svn/changes.py b/repoman/pym/repoman/modules/vcs/svn/changes.py deleted file mode 100644 index dfed1655b..000000000 --- a/repoman/pym/repoman/modules/vcs/svn/changes.py +++ /dev/null @@ -1,142 +0,0 @@ -''' -Subversion module Changes class submodule -''' - -from itertools import chain - -from repoman.modules.vcs.changes import ChangesBase -from repoman._subprocess import repoman_popen -from repoman._subprocess import repoman_getstatusoutput -from repoman.modules.vcs.vcs import vcs_files_to_cps -from repoman._portage import portage -from portage import os -from portage.output import green -from portage.package.ebuild.digestgen import digestgen - - -class Changes(ChangesBase): - '''Class object to scan and hold the resultant data - for all changes to process. - ''' - - vcs = 'svn' - - def __init__(self, options, repo_settings): - '''Class init - - @param options: the run time cli options - @param repo_settings: RepoSettings instance - ''' - super(Changes, self).__init__(options, repo_settings) - - def _scan(self): - '''VCS type scan function, looks for all detectable changes''' - with repoman_popen("svn status") as f: - svnstatus = f.readlines() - self.changed = [ - "./" + elem.split()[-1:][0] - for elem in svnstatus - if elem and elem[:1] in "MR"] - self.new = [ - "./" + elem.split()[-1:][0] - for elem in svnstatus - if elem.startswith("A")] - self.removed = [ - "./" + elem.split()[-1:][0] - for elem in svnstatus - if elem.startswith("D")] - - @property - def expansion(self): - '''VCS method of getting the expanded keywords in the repository''' - if self._expansion is not None: - return self._expansion - # Subversion expands keywords specified in svn:keywords properties. - with repoman_popen("svn propget -R svn:keywords") as f: - props = f.readlines() - self._expansion = dict( - ("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) - for prop in props if " - " in prop) - del props - return self._expansion - - @property - def unadded(self): - '''VCS method of getting the unadded files in the repository''' - if self._unadded is not None: - return self._unadded - with repoman_popen("svn status --no-ignore") as f: - svnstatus = f.readlines() - self._unadded = [ - "./" + elem.rstrip().split()[1] - for elem in svnstatus - if elem.startswith("?") or elem.startswith("I")] - del svnstatus - return self._unadded - - def thick_manifest(self, updates, headers, no_expansion, expansion): - '''Create a thick manifest - - @param updates: - @param headers: - @param no_expansion: - @param expansion: - ''' - svn_keywords = dict((k.lower(), k) for k in [ - "Rev", - "Revision", - "LastChangedRevision", - "Date", - "LastChangedDate", - "Author", - "LastChangedBy", - "URL", - "HeadURL", - "Id", - "Header", - ]) - - for _file in updates: - # for SVN, expansion contains files that are included in expansion - if _file not in expansion: - continue - - # Subversion keywords are case-insensitive - # in svn:keywords properties, - # but case-sensitive in contents of files. - enabled_keywords = [] - for k in expansion[_file]: - keyword = svn_keywords.get(k.lower()) - if keyword is not None: - enabled_keywords.append(keyword) - - headerstring = r"'\$(%s).*\$'" % "|".join(enabled_keywords) - - _out = repoman_getstatusoutput( - "egrep -q %s %s" % (headerstring, portage._shell_quote(_file))) - if _out[0] == 0: - headers.append(_file) - - print("%s have headers that will change." % green(str(len(headers)))) - print( - "* Files with headers will" - " cause the manifests to be changed and committed separately.") - - def digest_regen(self, updates, removed, manifests, scanner, broken_changelog_manifests): - '''Regenerate manifests - - @param updates: updated files - @param removed: removed files - @param manifests: Manifest files - @param scanner: The repoman.scanner.Scanner instance - @param broken_changelog_manifests: broken changelog manifests - ''' - if updates or removed: - for x in sorted(vcs_files_to_cps( - chain(updates, removed, manifests), - scanner.repo_settings.repodir, - scanner.repolevel, scanner.reposplit, scanner.categories)): - self.repoman_settings["O"] = os.path.join(self.repo_settings.repodir, x) - digestgen(mysettings=self.repoman_settings, myportdb=self.repo_settings.portdb) - - diff --git a/repoman/pym/repoman/modules/vcs/svn/status.py b/repoman/pym/repoman/modules/vcs/svn/status.py deleted file mode 100644 index 6575fe0b0..000000000 --- a/repoman/pym/repoman/modules/vcs/svn/status.py +++ /dev/null @@ -1,150 +0,0 @@ -''' -Subversion module Status class submodule -''' - -import logging -import subprocess -import sys - -from repoman._portage import portage -from portage import os -from portage.const import BASH_BINARY -from portage.output import red, green -from portage import _unicode_encode, _unicode_decode - -from repoman._subprocess import repoman_popen - - -class Status(object): - '''Performs status checks on the svn repository''' - - def __init__(self, qatracker, eadded): - '''Class init - - @param qatracker: QATracker class instance - @param eadded: list - ''' - self.qatracker = qatracker - self.eadded = eadded - - def check(self, checkdir, checkdir_relative, xpkg): - '''Perform the svn status check - - @param checkdir: string of the directory being checked - @param checkdir_relative: string of the relative directory being checked - @param xpkg: string of the package being checked - @returns: boolean - ''' - try: - myf = repoman_popen( - "svn status --depth=files --verbose " + - portage._shell_quote(checkdir)) - myl = myf.readlines() - myf.close() - except IOError: - raise - for l in myl: - if l[:1] == "?": - continue - if l[:7] == ' >': - # tree conflict, new in subversion 1.6 - continue - l = l.split()[-1] - if l[-7:] == ".ebuild": - self.eadded.append(os.path.basename(l[:-7])) - try: - myf = repoman_popen( - "svn status " + - portage._shell_quote(checkdir)) - myl = myf.readlines() - myf.close() - except IOError: - raise - for l in myl: - if l[0] == "A": - l = l.rstrip().split(' ')[-1] - if l[-7:] == ".ebuild": - self.eadded.append(os.path.basename(l[:-7])) - return True - - @staticmethod - def detect_conflicts(options): - """Determine if the checkout has problems like cvs conflicts. - - If you want more vcs support here just keep adding if blocks... - This could be better. - - TODO(antarus): Also this should probably not call sys.exit() as - repoman is run on >1 packages and one failure should not cause - subsequent packages to fail. - - Args: - vcs - A string identifying the version control system in use - Returns: boolean - (calls sys.exit on fatal problems) - """ - - cmd = "svn status -u 2>&1 | egrep -v '^. +.*/digest-[^/]+' | head -n-1" - msg = ("Performing a %s with a little magic grep to check for updates." - % green("svn status -u")) - - logging.info(msg) - # Use Popen instead of getstatusoutput(), in order to avoid - # unicode handling problems (see bug #310789). - args = [BASH_BINARY, "-c", cmd] - args = [_unicode_encode(x) for x in args] - proc = subprocess.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - out = _unicode_decode(proc.communicate()[0]) - proc.wait() - mylines = out.splitlines() - myupdates = [] - for line in mylines: - if not line: - continue - - # [ ] Unmodified (SVN) [U] Updates [P] Patches - # [M] Modified [A] Added [R] Removed / Replaced - # [D] Deleted - if line[0] not in " UPMARD": - # Stray Manifest is fine, we will readd it anyway. - if line[0] == '?' and line[1:].lstrip() == 'Manifest': - continue - logging.error(red( - "!!! Please fix the following issues reported " - "from cvs: %s" % green("(U,P,M,A,R,D are ok)"))) - logging.error(red( - "!!! Note: This is a pretend/no-modify pass...")) - logging.error(out) - sys.exit(1) - elif line[8] == '*': - myupdates.append(line[9:].lstrip(" 1234567890")) - - if myupdates: - logging.info(green("Fetching trivial updates...")) - if options.pretend: - logging.info("(svn update " + " ".join(myupdates) + ")") - retval = os.EX_OK - else: - retval = os.system("svn update " + " ".join(myupdates)) - if retval != os.EX_OK: - logging.fatal("!!! svn exited with an error. Terminating.") - sys.exit(retval) - return False - - @staticmethod - def supports_gpg_sign(): - '''Does this vcs system support gpg commit signatures - - @returns: Boolean - ''' - return False - - @staticmethod - def isVcsDir(dirname): - '''Does the directory belong to the vcs system - - @param dirname: string, directory name - @returns: Boolean - ''' - return dirname in [".svn"] diff --git a/repoman/pym/repoman/modules/vcs/vcs.py b/repoman/pym/repoman/modules/vcs/vcs.py deleted file mode 100644 index b4ff3dc9c..000000000 --- a/repoman/pym/repoman/modules/vcs/vcs.py +++ /dev/null @@ -1,161 +0,0 @@ -# -*- coding:utf-8 -*- - -from __future__ import print_function, unicode_literals - -import collections -import logging -from itertools import chain - -from portage import os - - -_vcs_type = collections.namedtuple('_vcs_type', 'name dir_name file_name') - -_FindVCS_data = ( - _vcs_type( - name='git', - dir_name='.git', - file_name='.git' - ), - _vcs_type( - name='bzr', - dir_name='.bzr', - file_name='' - ), - _vcs_type( - name='hg', - dir_name='.hg', - file_name='' - ), - _vcs_type( - name='svn', - dir_name='.svn', - file_name='' - ) -) - - -def FindVCS(cwd=None): - """ - Try to figure out in what VCS' working tree we are. - - @param cwd: working directory (default is os.getcwd()) - @type cwd: str - @return: list of strings describing the discovered vcs types - @rtype: list - """ - - if cwd is None: - cwd = os.getcwd() - - outvcs = [] - - def seek(depth=None): - '''Seek for VCSes that have a top-level data directory only. - - @param depth: integer - @returns: list of strings - ''' - retvcs = [] - pathprep = cwd - - while depth is None or depth > 0: - for vcs_type in _FindVCS_data: - vcs_dir = os.path.join(pathprep, vcs_type.dir_name) - if os.path.isdir(vcs_dir): - logging.debug( - 'FindVCS: found %(name)s dir: %(vcs_dir)s' % { - 'name': vcs_type.name, - 'vcs_dir': os.path.abspath(vcs_dir)}) - retvcs.append(vcs_type.name) - elif vcs_type.file_name: - vcs_file = os.path.join(pathprep, vcs_type.file_name) - if os.path.exists(vcs_file): - logging.debug( - 'FindVCS: found %(name)s file: %(vcs_file)s' % { - 'name': vcs_type.name, - 'vcs_file': os.path.abspath(vcs_file)}) - retvcs.append(vcs_type.name) - - if retvcs: - break - pathprep = os.path.join(pathprep, '..') - if os.path.realpath(pathprep).strip('/') == '': - break - if depth is not None: - depth = depth - 1 - - return retvcs - - # Level zero VCS-es. - if os.path.isdir(os.path.join(cwd, 'CVS')): - outvcs.append('cvs') - if os.path.isdir('.svn'): # <1.7 - outvcs.append(os.path.join(cwd, 'svn')) - - # If we already found one of 'level zeros', just take a quick look - # at the current directory. Otherwise, seek parents till we get - # something or reach root. - if outvcs: - outvcs.extend(seek(1)) - else: - outvcs = seek() - - if len(outvcs) > 1: - # eliminate duplicates, like for svn in bug #391199 - outvcs = list(set(outvcs)) - - return outvcs - - -def vcs_files_to_cps(vcs_file_iter, repodir, repolevel, reposplit, categories): - """ - Iterate over the given modified file paths returned from the vcs, - and return a frozenset containing category/pn strings for each - modified package. - """ - - modified_cps = [] - - if repolevel == 3: - if reposplit[-2] in categories and \ - next(vcs_file_iter, None) is not None: - modified_cps.append("/".join(reposplit[-2:])) - - elif repolevel == 2: - category = reposplit[-1] - if category in categories: - for filename in vcs_file_iter: - f_split = filename.split(os.sep) - # ['.', pn, ...] - if len(f_split) > 2: - modified_cps.append(category + "/" + f_split[1]) - - else: - # repolevel == 1 - for filename in vcs_file_iter: - f_split = filename.split(os.sep) - # ['.', category, pn, ...] - if len(f_split) > 3 and f_split[1] in categories: - modified_cps.append("/".join(f_split[1:3])) - - # Exclude packages that have been removed, since calling - # code assumes that the packages exist. - return frozenset(x for x in frozenset(modified_cps) - if os.path.exists(os.path.join(repodir, x))) - - -def vcs_new_changed(relative_path, mychanged, mynew): - '''Check if any vcs tracked file have been modified - - @param relative_path: - @param mychanged: iterable of changed files - @param mynew: iterable of new files - @returns boolean - ''' - for x in chain(mychanged, mynew): - if x == relative_path: - return True - return False - - diff --git a/repoman/pym/repoman/profile.py b/repoman/pym/repoman/profile.py deleted file mode 100644 index 50da91728..000000000 --- a/repoman/pym/repoman/profile.py +++ /dev/null @@ -1,87 +0,0 @@ -# -*- coding:utf-8 -*- - -from __future__ import print_function, unicode_literals - -from portage import normalize_path -from portage import os -from portage.output import red - - -class ProfileDesc(object): - __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',) - - def __init__(self, arch, status, sub_path, tree_path): - self.arch = arch - self.status = status - if sub_path: - sub_path = normalize_path(sub_path.lstrip(os.sep)) - self.sub_path = sub_path - self.tree_path = tree_path - if tree_path: - self.abs_path = os.path.join(tree_path, 'profiles', self.sub_path) - else: - self.abs_path = tree_path - - def __str__(self): - if self.sub_path: - return self.sub_path - return 'empty profile' - - -valid_profile_types = frozenset(['dev', 'exp', 'stable']) - - -def dev_profile_keywords(profiles): - """ - Create a set of KEYWORDS values that exist in 'dev' - profiles. These are used - to trigger a message notifying the user when they might - want to add the --include-dev option. - """ - type_arch_map = {} - for arch, arch_profiles in profiles.items(): - for prof in arch_profiles: - arch_set = type_arch_map.get(prof.status) - if arch_set is None: - arch_set = set() - type_arch_map[prof.status] = arch_set - arch_set.add(arch) - - dev_keywords = type_arch_map.get('dev', set()) - dev_keywords.update(['~' + arch for arch in dev_keywords]) - return frozenset(dev_keywords) - - -def setup_profile(profile_list): - # Ensure that profile sub_path attributes are unique. Process in reverse order - # so that profiles with duplicate sub_path from overlays will override - # profiles with the same sub_path from parent repos. - profiles = {} - profile_list.reverse() - profile_sub_paths = set() - for prof in profile_list: - if prof.sub_path in profile_sub_paths: - continue - profile_sub_paths.add(prof.sub_path) - profiles.setdefault(prof.arch, []).append(prof) - - # Use an empty profile for checking dependencies of - # packages that have empty KEYWORDS. - prof = ProfileDesc('**', 'stable', '', '') - profiles.setdefault(prof.arch, []).append(prof) - return profiles - - -def check_profiles(profiles, archlist): - for x in archlist: - if x[0] == "~": - continue - if x not in profiles: - print(red( - "\"%s\" doesn't have a valid profile listed in profiles.desc." % x)) - print(red( - "You need to either \"cvs update\" your profiles dir" - " or follow this")) - print(red( - "up with the " + x + " team.")) - print() diff --git a/repoman/pym/repoman/qa_data.py b/repoman/pym/repoman/qa_data.py deleted file mode 100644 index 01141a617..000000000 --- a/repoman/pym/repoman/qa_data.py +++ /dev/null @@ -1,192 +0,0 @@ -# -*- coding:utf-8 -*- - -import logging -import os - -from _emerge.Package import Package - -# import our initialized portage instance -from repoman import _not_installed -from repoman._portage import portage -from repoman.config import load_config - - -class QAData(object): - - def __init__(self): - # Create the main exported data variables - self.max_desc_len = None - self.allowed_filename_chars = None - self.qahelp = None - self.qacats = None - self.qawarnings = None - self.missingvars = None - self.allvars = None - self.valid_restrict = None - self.suspect_rdepend = None - self.suspect_virtual = None - self.ruby_deprecated = None - self.no_exec = None - - - def load_repo_config(self, repopaths, options, valid_versions): - '''Load the repository repoman qa_data.yml config - - @param repopaths: list of strings, The path of the repository being scanned - This could be a parent repository using the - repoman_masters layout.conf variable - ''' - # add our base qahelp - repository_modules = options.experimental_repository_modules == 'y' - if _not_installed: - cnfdir = os.path.realpath(os.path.join(os.path.dirname( - os.path.dirname(os.path.dirname(__file__))), 'cnf/qa_data')) - else: - cnfdir = os.path.join(portage.const.EPREFIX or '/', 'usr/share/repoman/qa_data') - repomanpaths = [os.path.join(cnfdir, _file_) for _file_ in os.listdir(cnfdir)] - logging.debug("QAData: cnfdir: %s, repomanpaths: %s", cnfdir, repomanpaths) - if repository_modules: - repopaths = [os.path.join(path,'qa_data.yaml') for path in repopaths] - elif _not_installed: - repopaths = [os.path.realpath(os.path.join(os.path.dirname( - os.path.dirname(os.path.dirname(__file__))), - 'cnf/repository/qa_data.yaml'))] - else: - repopaths = [os.path.join(portage.const.EPREFIX or '/', - 'usr/share/repoman/repository/qa_data.yaml')] - infopaths = repomanpaths + repopaths - - qadata = load_config(infopaths, None, valid_versions) - if qadata == {}: - logging.error("QAData: Failed to load a valid 'qa_data.yaml' file at paths: %s", infopaths) - return False - self.max_desc_len = qadata.get('max_description_length', 80) - self.allowed_filename_chars = qadata.get("allowed_filename_chars", "a-zA-Z0-9._-+:") - - self.qahelp = qadata["qahelp"] - logging.debug("qa_help primary keys: %s", sorted(self.qahelp)) - - self.qacats = [] - for x in sorted(self.qahelp): - for y in sorted(self.qahelp[x]): - self.qacats.append('.'.join([x, y])) - self.qacats.sort() - - self.qawarnings = set(qadata.get('qawarnings', [])) - if options.experimental_inherit == 'y': - # This is experimental, so it's non-fatal. - self.qawarnings.add("inherit.missing") - - self.missingvars = qadata.get("missingvars", []) - logging.debug("QAData: missingvars: %s", self.missingvars) - self.allvars = set(x for x in portage.auxdbkeys if not x.startswith("UNUSED_")) - self.allvars.update(Package.metadata_keys) - self.allvars = sorted(self.allvars) - - for x in self.missingvars: - x += ".missing" - if x not in self.qacats: - logging.warning('QAData: * missingvars values need to be added to qahelp ("%s")' % x) - self.qacats.append(x) - self.qawarnings.add(x) - - self.valid_restrict = frozenset(qadata.get("valid_restrict", [])) - - self.suspect_rdepend = frozenset(qadata.get("suspect_rdepend", [])) - - self.suspect_virtual = qadata.get("suspect_virtual", {}) - - self.ruby_deprecated = frozenset(qadata.get("ruby_deprecated", [])) - - # file.executable - self.no_exec = frozenset(qadata.get("no_exec_files", [])) - logging.debug("QAData: completed loading file: %s", repopaths) - return True - - -def format_qa_output( - formatter, fails, dofull, dofail, options, qawarnings): - """Helper function that formats output properly - - @param formatter: an instance of Formatter - @type formatter: Formatter - @param fails: dict of qa status failures - @type fails: dict - @param dofull: Whether to print full results or a summary - @type dofull: boolean - @param dofail: Whether failure was hard or soft - @type dofail: boolean - @param options: The command-line options provided to repoman - @type options: Namespace - @param qawarnings: the set of warning types - @type qawarnings: set - @return: None (modifies formatter) - """ - full = options.mode == 'full' - # we only want key value pairs where value > 0 - for category in sorted(fails): - number = len(fails[category]) - formatter.add_literal_data(" " + category) - spacing_width = 30 - len(category) - if category in qawarnings: - formatter.push_style("WARN") - else: - formatter.push_style("BAD") - formatter.add_literal_data(" [fatal]") - spacing_width -= 8 - - formatter.add_literal_data(" " * spacing_width) - formatter.add_literal_data("%s" % number) - formatter.pop_style() - formatter.add_line_break() - if not dofull: - if not full and dofail and category in qawarnings: - # warnings are considered noise when there are failures - continue - fails_list = fails[category] - if not full and len(fails_list) > 12: - fails_list = fails_list[:12] - for failure in fails_list: - formatter.add_literal_data(" " + failure) - formatter.add_line_break() - - -def format_qa_output_column( - formatter, fails, dofull, dofail, options, qawarnings): - """Helper function that formats output in a machine-parseable column format - - @param formatter: an instance of Formatter - @type formatter: Formatter - @param fails: dict of qa status failures - @type fails: dict - @param dofull: Whether to print full results or a summary - @type dofull: boolean - @param dofail: Whether failure was hard or soft - @type dofail: boolean - @param options: The command-line options provided to repoman - @type options: Namespace - @param qawarnings: the set of warning types - @type qawarnings: set - @return: None (modifies formatter) - """ - full = options.mode == 'full' - for category in sorted(fails): - number = len(fails[category]) - formatter.add_literal_data("NumberOf " + category + " ") - if category in qawarnings: - formatter.push_style("WARN") - else: - formatter.push_style("BAD") - formatter.add_literal_data("%s" % number) - formatter.pop_style() - formatter.add_line_break() - if not dofull: - if not full and dofail and category in qawarnings: - # warnings are considered noise when there are failures - continue - fails_list = fails[category] - if not full and len(fails_list) > 12: - fails_list = fails_list[:12] - for failure in fails_list: - formatter.add_literal_data(category + " " + failure) - formatter.add_line_break() diff --git a/repoman/pym/repoman/qa_tracker.py b/repoman/pym/repoman/qa_tracker.py deleted file mode 100644 index faaf8e398..000000000 --- a/repoman/pym/repoman/qa_tracker.py +++ /dev/null @@ -1,45 +0,0 @@ - -import logging -import sys - - -class QATracker(object): - '''Track all occurrances of Q/A problems detected''' - - def __init__(self, qacats=None, qawarnings=None): - self.fails = {} - self.warns = {} - self.qacats = qacats - self.qawarnings = qawarnings - - def add_error(self, detected_qa, info): - '''Add the Q/A error to the database of detected problems - - @param detected_qa: string, member of qa_data.qacats list - @param info: string, details of the detected problem - ''' - if detected_qa not in self.qacats: - logging.error( - 'QATracker: Exiting on error. Unknown detected_qa type passed ' - 'in to add_error(): %s, %s' % (detected_qa, info)) - sys.exit(1) - try: - self.fails[detected_qa].append(info) - except KeyError: - self.fails[detected_qa] = [info] - - def add_warning(self, detected_qa, info): - '''Add the Q/A warning to the database of detected problems - - @param detected_qa: string, member of qa_data.qawarnings list - @param info: string, details of the detected problem - ''' - if detected_qa not in self.qawarnings: - logging.error( - 'QATracker: Exiting on error. Unknown detected_qa type passed ' - 'in to add_warning(): %s, %s' % (detected_qa, info)) - sys.exit(1) - try: - self.warns[detected_qa].append(info) - except KeyError: - self.warns[detected_qa] = [info] diff --git a/repoman/pym/repoman/repos.py b/repoman/pym/repoman/repos.py deleted file mode 100644 index 9c820b08c..000000000 --- a/repoman/pym/repoman/repos.py +++ /dev/null @@ -1,316 +0,0 @@ -# -*- coding:utf-8 -*- - - -import io -import logging -import re -import sys -import textwrap - -# import our initialized portage instance -from repoman._portage import portage - -from portage import os -from portage import _encodings -from portage import _unicode_encode -from portage.checksum import get_valid_checksum_keys - -from repoman.errors import err -from repoman.profile import ProfileDesc, valid_profile_types - -GPG_KEY_ID_REGEX = r'(0x)?([0-9a-fA-F]{8}){1,5}!?' -bad = portage.output.create_color_func("BAD") - - -class RepoSettings(object): - '''Holds our repo specific settings''' - - def __init__( - self, config_root, portdir, portdir_overlay, - repoman_settings=None, vcs_settings=None, options=None, - qadata=None): - self.config_root = config_root - self.repoman_settings = repoman_settings - self.vcs_settings = vcs_settings - - self.repositories = self.repoman_settings.repositories - - # Ensure that current repository is in the list of enabled repositories. - self.repodir = os.path.realpath(portdir_overlay) - try: - self.repositories.get_repo_for_location(self.repodir) - except KeyError: - self._add_repo(config_root, portdir_overlay) - - # Determine the master config loading list - self.masters_list = [] - # get out repo masters value - masters = self.repositories.get_repo_for_location(self.repodir).masters - for repo in masters: - self.masters_list.append(os.path.join(repo.location, 'metadata', 'repoman')) - self.masters_list.append(os.path.join(self.repodir, 'metadata', 'repoman')) - - logging.debug("RepoSettings: init(); load qadata") - # load the repo specific configuration - self.qadata = qadata - if not self.qadata.load_repo_config(self.masters_list, options, repoman_settings.valid_versions): - logging.error("Aborting...") - sys.exit(1) - logging.debug("RepoSettings: qadata loaded: %s", qadata.no_exec) - - self.root = self.repoman_settings['EROOT'] - self.trees = { - self.root: {'porttree': portage.portagetree(settings=self.repoman_settings)} - } - self.portdb = self.trees[self.root]['porttree'].dbapi - - # Constrain dependency resolution to the master(s) - # that are specified in layout.conf. - self.repo_config = self.repositories.get_repo_for_location(self.repodir) - self.portdb.porttrees = list(self.repo_config.eclass_db.porttrees) - self.portdir = self.portdb.porttrees[0] - self.commit_env = os.environ.copy() - # list() is for iteration on a copy. - for repo in list(self.repositories): - # all paths are canonical - if repo.location not in self.repo_config.eclass_db.porttrees: - del self.repositories[repo.name] - - if self.repo_config.sign_commit and options.mode in ("commit", "fix", "manifest"): - if vcs_settings.vcs: - func = getattr(self, '_vcs_gpg_%s' % vcs_settings.vcs) - func() - else: - logging.warning("No VCS type detected, unable to sign the commit") - - # In order to disable manifest signatures, repos may set - # "sign-manifests = false" in metadata/layout.conf. This - # can be used to prevent merge conflicts like those that - # thin-manifests is designed to prevent. - self.sign_manifests = "sign" in self.repoman_settings.features and \ - self.repo_config.sign_manifest - - if self.repo_config.sign_manifest and self.repo_config.name == "gentoo" and \ - options.mode in ("commit",) and not self.sign_manifests: - msg = ( - "The '%s' repository has manifest signatures enabled, " - "but FEATURES=sign is currently disabled. In order to avoid this " - "warning, enable FEATURES=sign in make.conf. Alternatively, " - "repositories can disable manifest signatures by setting " - "'sign-manifests = false' in metadata/layout.conf.") % ( - self.repo_config.name,) - for line in textwrap.wrap(msg, 60): - logging.warn(line) - - is_commit = options.mode in ("commit",) - valid_gpg_key = self.repoman_settings.get("PORTAGE_GPG_KEY") and re.match( - r'^%s$' % GPG_KEY_ID_REGEX, self.repoman_settings["PORTAGE_GPG_KEY"]) - - if self.sign_manifests and is_commit and not valid_gpg_key: - logging.error( - "PORTAGE_GPG_KEY value is invalid: %s" % - self.repoman_settings["PORTAGE_GPG_KEY"]) - sys.exit(1) - - manifest_hashes = self.repo_config.manifest_hashes - manifest_required_hashes = self.repo_config.manifest_required_hashes - if manifest_hashes is None: - manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS - manifest_required_hashes = manifest_hashes - - if options.mode in ("commit", "fix", "manifest"): - missing_required_hashes = manifest_required_hashes.difference( - manifest_hashes) - if missing_required_hashes: - msg = ( - "The 'manifest-hashes' setting in the '%s' repository's " - "metadata/layout.conf does not contain the '%s' hashes which " - "are listed in 'manifest-required-hashes'. Please fix that " - "file if you want to generate valid manifests for " - "this repository.") % ( - self.repo_config.name, ' '.join(missing_required_hashes)) - for line in textwrap.wrap(msg, 70): - logging.error(line) - sys.exit(1) - - unsupported_hashes = manifest_hashes.difference( - get_valid_checksum_keys()) - if unsupported_hashes: - msg = ( - "The 'manifest-hashes' setting in the '%s' repository's " - "metadata/layout.conf contains one or more hash types '%s' " - "which are not supported by this portage version. You will " - "have to upgrade portage if you want to generate valid " - "manifests for this repository.") % ( - self.repo_config.name, " ".join(sorted(unsupported_hashes))) - for line in textwrap.wrap(msg, 70): - logging.error(line) - sys.exit(1) - - def _add_repo(self, config_root, portdir_overlay): - self.repo_conf = portage.repository.config - self.repo_name = self.repo_conf.RepoConfig._read_valid_repo_name( - portdir_overlay)[0] - self.layout_conf_data = self.repo_conf.parse_layout_conf(portdir_overlay)[0] - if self.layout_conf_data['repo-name']: - self.repo_name = self.layout_conf_data['repo-name'] - tmp_conf_file = io.StringIO(textwrap.dedent(""" - [%s] - location = %s - """) % (self.repo_name, portdir_overlay)) - # Ensure that the repository corresponding to $PWD overrides a - # repository of the same name referenced by the existing PORTDIR - # or PORTDIR_OVERLAY settings. - self.repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % ( - self.repoman_settings.get('PORTDIR_OVERLAY', ''), - portage._shell_quote(portdir_overlay)) - self.repositories = self.repo_conf.load_repository_config( - self.repoman_settings, extra_files=[tmp_conf_file]) - # We have to call the config constructor again so that attributes - # dependent on config.repositories are initialized correctly. - self.repoman_settings = portage.config( - config_root=config_root, local_config=False, - repositories=self.repositories) - - ########## - # future vcs plugin functions - ########## - - def _vcs_gpg_bzr(self): - pass - - def _vcs_gpg_cvs(self): - pass - - def _vcs_gpg_git(self): - # NOTE: It's possible to use --gpg-sign=key_id to specify the key in - # the commit arguments. If key_id is unspecified, then it must be - # configured by `git config user.signingkey key_id`. - self.vcs_settings.vcs_local_opts.append("--gpg-sign") - if self.repoman_settings.get("PORTAGE_GPG_DIR"): - # Pass GNUPGHOME to git for bug #462362. - self.commit_env["GNUPGHOME"] = self.repoman_settings["PORTAGE_GPG_DIR"] - - # Pass GPG_TTY to git for bug #477728. - try: - self.commit_env["GPG_TTY"] = os.ttyname(sys.stdin.fileno()) - except OSError: - pass - - def _vcs_gpg_hg(self): - pass - - def _vcs_gpg_svn(self): - pass - - -def list_checks(kwlist, liclist, uselist, repoman_settings): - liclist_deprecated = set() - if "DEPRECATED" in repoman_settings._license_manager._license_groups: - liclist_deprecated.update( - repoman_settings._license_manager.expandLicenseTokens(["@DEPRECATED"])) - - if not liclist: - logging.fatal("Couldn't find licenses?") - sys.exit(1) - - if not kwlist: - logging.fatal("Couldn't read KEYWORDS from arch.list") - sys.exit(1) - - if not uselist: - logging.fatal("Couldn't find use.desc?") - sys.exit(1) - return liclist_deprecated - - -def repo_metadata(portdb, repoman_settings): - # get lists of valid keywords, licenses, and use - kwlist = set() - liclist = set() - uselist = set() - profile_list = [] - global_pmasklines = [] - - for path in portdb.porttrees: - try: - liclist.update(os.listdir(os.path.join(path, "licenses"))) - except OSError: - pass - kwlist.update( - portage.grabfile(os.path.join(path, "profiles", "arch.list"))) - - use_desc = portage.grabfile(os.path.join(path, 'profiles', 'use.desc')) - for x in use_desc: - x = x.split() - if x: - uselist.add(x[0]) - - expand_desc_dir = os.path.join(path, 'profiles', 'desc') - try: - expand_list = os.listdir(expand_desc_dir) - except OSError: - pass - else: - for fn in expand_list: - if not fn[-5:] == '.desc': - continue - use_prefix = fn[:-5].lower() + '_' - for x in portage.grabfile(os.path.join(expand_desc_dir, fn)): - x = x.split() - if x: - uselist.add(use_prefix + x[0]) - - global_pmasklines.append( - portage.util.grabfile_package( - os.path.join(path, 'profiles', 'package.mask'), - recursive=1, verify_eapi=True)) - - desc_path = os.path.join(path, 'profiles', 'profiles.desc') - try: - desc_file = io.open( - _unicode_encode( - desc_path, encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], errors='replace') - except EnvironmentError: - pass - else: - for i, x in enumerate(desc_file): - if x[0] == "#": - continue - arch = x.split() - if len(arch) == 0: - continue - if len(arch) != 3: - err( - "wrong format: \"%s\" in %s line %d" % - (bad(x.strip()), desc_path, i + 1, )) - elif arch[0] not in kwlist: - err( - "invalid arch: \"%s\" in %s line %d" % - (bad(arch[0]), desc_path, i + 1, )) - elif arch[2] not in valid_profile_types: - err( - "invalid profile type: \"%s\" in %s line %d" % - (bad(arch[2]), desc_path, i + 1, )) - profile_desc = ProfileDesc(arch[0], arch[2], arch[1], path) - if not os.path.isdir(profile_desc.abs_path): - logging.error( - "Invalid %s profile (%s) for arch %s in %s line %d", - arch[2], arch[1], arch[0], desc_path, i + 1) - continue - if os.path.exists( - os.path.join(profile_desc.abs_path, 'deprecated')): - continue - profile_list.append(profile_desc) - desc_file.close() - - global_pmasklines = portage.util.stack_lists(global_pmasklines, incremental=1) - global_pmaskdict = {} - for x in global_pmasklines: - global_pmaskdict.setdefault(x.cp, []).append(x) - del global_pmasklines - - return ( - kwlist, liclist, uselist, profile_list, global_pmaskdict, - list_checks(kwlist, liclist, uselist, repoman_settings)) diff --git a/repoman/pym/repoman/scanner.py b/repoman/pym/repoman/scanner.py deleted file mode 100644 index c456bbde9..000000000 --- a/repoman/pym/repoman/scanner.py +++ /dev/null @@ -1,422 +0,0 @@ -# -*- coding:utf-8 -*- - -from __future__ import print_function, unicode_literals - -import logging -from itertools import chain - -import portage -from portage import normalize_path -from portage import os -from portage.output import green -from portage.util.futures.extendedfutures import ExtendedFuture -from repoman.metadata import get_metadata_xsd -from repoman.modules.commit import repochecks -from repoman.modules.commit import manifest -from repoman.profile import check_profiles, dev_profile_keywords, setup_profile -from repoman.repos import repo_metadata -from repoman.modules.scan.module import ModuleConfig -from repoman.modules.scan.scan import scan -from repoman.modules.vcs.vcs import vcs_files_to_cps - - -DATA_TYPES = {'dict': dict, 'Future': ExtendedFuture, 'list': list, 'set': set} - - -class Scanner(object): - '''Primary scan class. Operates all the small Q/A tests and checks''' - - def __init__(self, repo_settings, myreporoot, config_root, options, - vcs_settings, mydir, env): - '''Class __init__''' - self.repo_settings = repo_settings - self.config_root = config_root - self.options = options - self.vcs_settings = vcs_settings - self.env = env - - # Repoman sets it's own ACCEPT_KEYWORDS and we don't want it to - # behave incrementally. - self.repoman_incrementals = tuple( - x for x in portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS') - - self.categories = [] - for path in self.repo_settings.repo_config.eclass_db.porttrees: - self.categories.extend(portage.util.grabfile( - os.path.join(path, 'profiles', 'categories'))) - self.repo_settings.repoman_settings.categories = frozenset( - portage.util.stack_lists([self.categories], incremental=1)) - self.categories = self.repo_settings.repoman_settings.categories - - self.portdb = repo_settings.portdb - self.portdb.settings = self.repo_settings.repoman_settings - - digest_only = self.options.mode != 'manifest-check' \ - and self.options.digest == 'y' - self.generate_manifest = digest_only or self.options.mode in \ - ("manifest", 'commit', 'fix') - - # We really only need to cache the metadata that's necessary for visibility - # filtering. Anything else can be discarded to reduce memory consumption. - if not self.generate_manifest: - # Don't do this when generating manifests, since that uses - # additional keys if spawn_nofetch is called (RESTRICT and - # DEFINED_PHASES). - self.portdb._aux_cache_keys.clear() - self.portdb._aux_cache_keys.update( - ["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"]) - - self.reposplit = myreporoot.split(os.path.sep) - self.repolevel = len(self.reposplit) - - if self.options.mode == 'commit': - repochecks.commit_check(self.repolevel, self.reposplit) - repochecks.conflict_check(self.vcs_settings, self.options) - - # Make startdir relative to the canonical repodir, so that we can pass - # it to digestgen and it won't have to be canonicalized again. - if self.repolevel == 1: - startdir = self.repo_settings.repodir - else: - startdir = normalize_path(mydir) - startdir = os.path.join( - self.repo_settings.repodir, *startdir.split(os.sep)[-2 - self.repolevel + 3:]) - - # get lists of valid keywords, licenses, and use - new_data = repo_metadata(self.portdb, self.repo_settings.repoman_settings) - kwlist, liclist, uselist, profile_list, \ - global_pmaskdict, liclist_deprecated = new_data - self.repo_metadata = { - 'kwlist': kwlist, - 'liclist': liclist, - 'uselist': uselist, - 'profile_list': profile_list, - 'pmaskdict': global_pmaskdict, - 'lic_deprecated': liclist_deprecated, - } - - self.repo_settings.repoman_settings['PORTAGE_ARCHLIST'] = ' '.join(sorted(kwlist)) - self.repo_settings.repoman_settings.backup_changes('PORTAGE_ARCHLIST') - - profiles = setup_profile(profile_list) - - check_profiles(profiles, self.repo_settings.repoman_settings.archlist()) - - scanlist = scan(self.repolevel, self.reposplit, startdir, self.categories, self.repo_settings) - - self.dev_keywords = dev_profile_keywords(profiles) - - self.qatracker = self.vcs_settings.qatracker - - if self.options.echangelog is None and self.repo_settings.repo_config.update_changelog: - self.options.echangelog = 'y' - - if self.vcs_settings.vcs is None: - self.options.echangelog = 'n' - - # Initialize the ModuleConfig class here - # TODO Add layout.conf masters repository.yml config to the list to load/stack - self.moduleconfig = ModuleConfig(self.repo_settings.masters_list, - self.repo_settings.repoman_settings.valid_versions, - repository_modules=self.options.experimental_repository_modules == 'y') - - checks = {} - # The --echangelog option causes automatic ChangeLog generation, - # which invalidates changelog.ebuildadded and changelog.missing - # checks. - # Note: Some don't use ChangeLogs in distributed SCMs. - # It will be generated on server side from scm log, - # before package moves to the rsync server. - # This is needed because they try to avoid merge collisions. - # Gentoo's Council decided to always use the ChangeLog file. - # TODO: shouldn't this just be switched on the repo, iso the VCS? - is_echangelog_enabled = self.options.echangelog in ('y', 'force') - self.vcs_settings.vcs_is_cvs_or_svn = self.vcs_settings.vcs in ('cvs', 'svn') - checks['changelog'] = not is_echangelog_enabled and self.vcs_settings.vcs_is_cvs_or_svn - - if self.options.mode == "manifest" or self.options.quiet: - pass - elif self.options.pretend: - print(green("\nRepoMan does a once-over of the neighborhood...")) - else: - print(green("\nRepoMan scours the neighborhood...")) - - self.changed = self.vcs_settings.changes - # bypass unneeded VCS operations if not needed - if (self.options.if_modified == "y" or - self.options.mode not in ("manifest", "manifest-check")): - self.changed.scan() - - self.have = { - 'pmasked': False, - 'dev_keywords': False, - } - - # NOTE: match-all caches are not shared due to potential - # differences between profiles in _get_implicit_iuse. - self.caches = { - 'arch': {}, - 'arch_xmatch': {}, - 'shared_xmatch': {"cp-list": {}}, - } - - self.include_arches = None - if self.options.include_arches: - self.include_arches = set() - self.include_arches.update(*[x.split() for x in self.options.include_arches]) - - # Disable the "self.modules['Ebuild'].notadded" check when not in commit mode and - # running `svn status` in every package dir will be too expensive. - checks['ebuild_notadded'] = not \ - (self.vcs_settings.vcs == "svn" and self.repolevel < 3 and self.options.mode != "commit") - - self.effective_scanlist = scanlist - if self.options.if_modified == "y": - self.effective_scanlist = sorted(vcs_files_to_cps( - chain(self.changed.changed, self.changed.new, self.changed.removed), - self.repo_settings.repodir, - self.repolevel, self.reposplit, self.categories)) - - # Create our kwargs dict here to initialize the plugins with - self.kwargs = { - "repo_settings": self.repo_settings, - "portdb": self.portdb, - "qatracker": self.qatracker, - "vcs_settings": self.vcs_settings, - "options": self.options, - "metadata_xsd": get_metadata_xsd(self.repo_settings), - "uselist": uselist, - "checks": checks, - "repo_metadata": self.repo_metadata, - "profiles": profiles, - "include_arches": self.include_arches, - "caches": self.caches, - "repoman_incrementals": self.repoman_incrementals, - "env": self.env, - "have": self.have, - "dev_keywords": self.dev_keywords, - "linechecks": self.moduleconfig.linechecks, - } - # initialize the plugin checks here - self.modules = {} - self._ext_futures = {} - self.pkg_level_futures = None - - def set_kwargs(self, mod): - '''Creates a limited set of kwargs to pass to the module's __init__() - - @param mod: module name string - @returns: dictionary - ''' - kwargs = {} - for key in self.moduleconfig.controller.modules[mod]['mod_kwargs']: - kwargs[key] = self.kwargs[key] - return kwargs - - def set_func_kwargs(self, mod, dynamic_data=None): - '''Updates the dynamic_data dictionary with any new key, value pairs. - Creates a limited set of kwargs to pass to the modulefunctions to run - - @param mod: module name string - @param dynamic_data: dictionary structure - @returns: dictionary - ''' - func_kwargs = self.moduleconfig.controller.modules[mod]['func_kwargs'] - # determine new keys - required = set(list(func_kwargs)) - exist = set(list(dynamic_data)) - new = required.difference(exist) - # update dynamic_data with initialized entries - for key in new: - logging.debug("set_func_kwargs(); adding: %s, %s", - key, func_kwargs[key]) - if func_kwargs[key][0] in ['Future', 'ExtendedFuture']: - if key not in self._ext_futures: - logging.debug( - "Adding a new key: %s to the ExtendedFuture dict", key) - self._ext_futures[key] = func_kwargs[key] - self._set_future(dynamic_data, key, func_kwargs[key]) - else: # builtin python data type - dynamic_data[key] = DATA_TYPES[func_kwargs[key][0]]() - kwargs = {} - for key in required: - kwargs[key] = dynamic_data[key] - return kwargs - - def reset_futures(self, dynamic_data): - '''Reset any Future data types - - @param dynamic_data: dictionary - ''' - for key in list(self._ext_futures): - if key not in self.pkg_level_futures: - self._set_future(dynamic_data, key, self._ext_futures[key]) - - @staticmethod - def _set_future(dynamic_data, key, data): - '''Set a dynamic_data key to a new ExtendedFuture instance - - @param dynamic_data: dictionary - @param key: tuple of (dictionary-key, default-value) - ''' - if data[0] in ['Future', 'ExtendedFuture']: - if data[1] in ['UNSET']: - dynamic_data[key] = ExtendedFuture() - else: - if data[1] in DATA_TYPES: - default = DATA_TYPES[data[1]]() - else: - default = data[1] - dynamic_data[key] = ExtendedFuture(default) - - def scan_pkgs(self, can_force): - for xpkg in self.effective_scanlist: - xpkg_continue = False - # ebuilds and digests added to cvs respectively. - logging.info("checking package %s", xpkg) - # save memory by discarding xmatch caches from previous package(s) - self.caches['arch_xmatch'].clear() - catdir, pkgdir = xpkg.split("/") - checkdir = self.repo_settings.repodir + "/" + xpkg - checkdir_relative = "" - if self.repolevel < 3: - checkdir_relative = os.path.join(pkgdir, checkdir_relative) - if self.repolevel < 2: - checkdir_relative = os.path.join(catdir, checkdir_relative) - checkdir_relative = os.path.join(".", checkdir_relative) - - # Run the status check - if self.kwargs['checks']['ebuild_notadded']: - self.vcs_settings.status.check(checkdir, checkdir_relative, xpkg) - - if self.generate_manifest: - manifest.Manifest(**self.kwargs).update_manifest(checkdir) - if self.options.mode == 'manifest': - continue - checkdirlist = os.listdir(checkdir) - - dynamic_data = { - 'changelog_modified': False, - 'checkdirlist': ExtendedFuture(checkdirlist), - 'checkdir': checkdir, - 'xpkg': xpkg, - 'changed': self.changed, - 'checkdir_relative': checkdir_relative, - 'can_force': can_force, - 'repolevel': self.repolevel, - 'catdir': catdir, - 'pkgdir': pkgdir, - 'validity_future': ExtendedFuture(True), - 'y_ebuild': None, - # this needs to be reset at the pkg level only, - # easiest is to just initialize it here - 'muselist': ExtendedFuture(set()), - 'src_uri_error': ExtendedFuture(), - } - self.pkg_level_futures = [ - 'checkdirlist', - 'muselist', - 'pkgs', - 'src_uri_error', - 'validity_future', - ] - # need to set it up for ==> self.modules or some other ordered list - logging.debug("***** starting pkgs_loop: %s", self.moduleconfig.pkgs_loop) - for mod in self.moduleconfig.pkgs_loop: - mod_class = self.moduleconfig.controller.get_class(mod) - logging.debug("Initializing class name: %s", mod_class.__name__) - self.modules[mod_class.__name__] = mod_class(**self.set_kwargs(mod)) - logging.debug("scan_pkgs; module: %s", mod_class.__name__) - do_it, functions = self.modules[mod_class.__name__].runInPkgs - if do_it: - for func in functions: - _continue = func(**self.set_func_kwargs(mod, dynamic_data)) - if _continue: - # If we can't access all the metadata then it's totally unsafe to - # commit since there's no way to generate a correct Manifest. - # Do not try to do any more QA checks on this package since missing - # metadata leads to false positives for several checks, and false - # positives confuse users. - xpkg_continue = True - break - - if xpkg_continue: - continue - - # Sort ebuilds in ascending order for the KEYWORDS.dropped check. - pkgs = dynamic_data['pkgs'].get() - ebuildlist = sorted(pkgs.values()) - ebuildlist = [pkg.pf for pkg in ebuildlist] - - if self.kwargs['checks']['changelog'] and "ChangeLog" not in checkdirlist: - self.qatracker.add_error("changelog.missing", xpkg + "/ChangeLog") - - changelog_path = os.path.join(checkdir_relative, "ChangeLog") - dynamic_data["changelog_modified"] = changelog_path in self.changed.changelogs - - self._scan_ebuilds(ebuildlist, dynamic_data) - return - - - def _scan_ebuilds(self, ebuildlist, dynamic_data): - - for y_ebuild in ebuildlist: - self.reset_futures(dynamic_data) - dynamic_data['y_ebuild'] = y_ebuild - y_ebuild_continue = False - - # initialize per ebuild plugin checks here - # need to set it up for ==> self.modules_list or some other ordered list - for mod in self.moduleconfig.ebuilds_loop: - if mod: - mod_class = self.moduleconfig.controller.get_class(mod) - if mod_class.__name__ not in self.modules: - logging.debug("Initializing class name: %s", mod_class.__name__) - self.modules[mod_class.__name__] = mod_class(**self.set_kwargs(mod)) - logging.debug("scan_ebuilds: module: %s", mod_class.__name__) - do_it, functions = self.modules[mod_class.__name__].runInEbuilds - logging.debug("do_it: %s, functions: %s", do_it, [x.__name__ for x in functions]) - if do_it: - for func in functions: - logging.debug("\tRunning function: %s", func) - _continue = func(**self.set_func_kwargs(mod, dynamic_data)) - if _continue: - # If we can't access all the metadata then it's totally unsafe to - # commit since there's no way to generate a correct Manifest. - # Do not try to do any more QA checks on this package since missing - # metadata leads to false positives for several checks, and false - # positives confuse users. - y_ebuild_continue = True - # logging.debug("\t>>> Continuing") - break - - if y_ebuild_continue: - continue - - logging.debug("Finished ebuild plugin loop, continuing...") - - # Final checks - # initialize per pkg plugin final checks here - # need to set it up for ==> self.modules_list or some other ordered list - xpkg_complete = False - for mod in self.moduleconfig.final_loop: - if mod: - mod_class = self.moduleconfig.controller.get_class(mod) - if mod_class.__name__ not in self.modules: - logging.debug("Initializing class name: %s", mod_class.__name__) - self.modules[mod_class.__name__] = mod_class(**self.set_kwargs(mod)) - logging.debug("scan_ebuilds final checks: module: %s", mod_class.__name__) - do_it, functions = self.modules[mod_class.__name__].runInFinal - logging.debug("do_it: %s, functions: %s", do_it, [x.__name__ for x in functions]) - if do_it: - for func in functions: - logging.debug("\tRunning function: %s", func) - _continue = func(**self.set_func_kwargs(mod, dynamic_data)) - if _continue: - xpkg_complete = True - # logging.debug("\t>>> Continuing") - break - - if xpkg_complete: - return - return diff --git a/repoman/pym/repoman/tests/__init__.py b/repoman/pym/repoman/tests/__init__.py deleted file mode 100644 index d57ca9b10..000000000 --- a/repoman/pym/repoman/tests/__init__.py +++ /dev/null @@ -1,357 +0,0 @@ -# tests/__init__.py -- Portage Unit Test functionality -# Copyright 2006-2013 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from __future__ import print_function - -import argparse -import sys -import time -import unittest - -try: - from unittest.runner import _TextTestResult # new in python-2.7 -except ImportError: - from unittest import _TextTestResult - -try: - # They added the skip framework to python-2.7. - # Drop this once we drop python-2.6 support. - unittest_skip_shims = False - import unittest.SkipTest as SkipTest # new in python-2.7 -except ImportError: - unittest_skip_shims = True - -import repoman -from repoman import REPOMAN_BASE_PATH -from repoman._portage import portage - -from portage import os -from portage import _encodings -from portage import _unicode_decode -from portage.const import EPREFIX, GLOBAL_CONFIG_PATH, PORTAGE_BIN_PATH - -if repoman._not_installed: - cnf_path = os.path.join(REPOMAN_BASE_PATH, 'cnf') - cnf_path_repoman = cnf_path - cnf_etc_path = cnf_path - cnf_bindir = os.path.join(REPOMAN_BASE_PATH, 'bin') - cnf_sbindir = cnf_bindir -else: - cnf_path = os.path.join(EPREFIX or '/', GLOBAL_CONFIG_PATH) - cnf_path_repoman = os.path.join(EPREFIX or '/', - sys.prefix.lstrip(os.sep), 'share', 'repoman') - cnf_etc_path = os.path.join(EPREFIX or '/', 'etc') - cnf_eprefix = EPREFIX - cnf_bindir = os.path.join(EPREFIX or '/', 'usr', 'bin') - cnf_sbindir = os.path.join(EPREFIX or '/', 'usr', 'sbin') - - -def main(): - suite = unittest.TestSuite() - basedir = os.path.dirname(os.path.realpath(__file__)) - - usage = "usage: %s [options] [tests to run]" % os.path.basename(sys.argv[0]) - parser = argparse.ArgumentParser(usage=usage) - parser.add_argument("-l", "--list", help="list all tests", - action="store_true", dest="list_tests") - options, args = parser.parse_known_args(args=sys.argv) - - if (os.environ.get('NOCOLOR') in ('yes', 'true') or - os.environ.get('TERM') == 'dumb' or - not sys.stdout.isatty()): - portage.output.nocolor() - - if options.list_tests: - testdir = os.path.dirname(sys.argv[0]) - for mydir in getTestDirs(basedir): - testsubdir = os.path.basename(mydir) - for name in getTestNames(mydir): - print("%s/%s/%s.py" % (testdir, testsubdir, name)) - return os.EX_OK - - if len(args) > 1: - suite.addTests(getTestFromCommandLine(args[1:], basedir)) - else: - for mydir in getTestDirs(basedir): - suite.addTests(getTests(os.path.join(basedir, mydir), basedir)) - - result = TextTestRunner(verbosity=2).run(suite) - if not result.wasSuccessful(): - return 1 - return os.EX_OK - -def my_import(name): - mod = __import__(name) - components = name.split('.') - for comp in components[1:]: - mod = getattr(mod, comp) - return mod - -def getTestFromCommandLine(args, base_path): - result = [] - for arg in args: - realpath = os.path.realpath(arg) - path = os.path.dirname(realpath) - f = realpath[len(path)+1:] - - if not f.startswith("test") or not f.endswith(".py"): - raise Exception("Invalid argument: '%s'" % arg) - - mymodule = f[:-3] - result.extend(getTestsFromFiles(path, base_path, [mymodule])) - return result - -def getTestDirs(base_path): - TEST_FILE = b'__test__.py' - testDirs = [] - - # the os.walk help mentions relative paths as being quirky - # I was tired of adding dirs to the list, so now we add __test__.py - # to each dir we want tested. - for root, dirs, files in os.walk(base_path): - try: - root = _unicode_decode(root, - encoding=_encodings['fs'], errors='strict') - except UnicodeDecodeError: - continue - - if TEST_FILE in files: - testDirs.append(root) - - testDirs.sort() - return testDirs - -def getTestNames(path): - files = os.listdir(path) - files = [f[:-3] for f in files if f.startswith("test") and f.endswith(".py")] - files.sort() - return files - -def getTestsFromFiles(path, base_path, files): - parent_path = path[len(base_path)+1:] - parent_module = ".".join(("repoman", "tests", parent_path)) - parent_module = parent_module.replace('/', '.') - result = [] - for mymodule in files: - # Make the trailing / a . for module importing - modname = ".".join((parent_module, mymodule)) - mod = my_import(modname) - result.append(unittest.TestLoader().loadTestsFromModule(mod)) - return result - -def getTests(path, base_path): - """ - - path is the path to a given subdir ( 'portage/' for example) - This does a simple filter on files in that dir to give us modules - to import - - """ - return getTestsFromFiles(path, base_path, getTestNames(path)) - -class TextTestResult(_TextTestResult): - """ - We need a subclass of unittest._TextTestResult to handle tests with TODO - - This just adds an addTodo method that can be used to add tests - that are marked TODO; these can be displayed later - by the test runner. - """ - - def __init__(self, stream, descriptions, verbosity): - super(TextTestResult, self).__init__(stream, descriptions, verbosity) - self.todoed = [] - self.portage_skipped = [] - - def addTodo(self, test, info): - self.todoed.append((test, info)) - if self.showAll: - self.stream.writeln("TODO") - elif self.dots: - self.stream.write(".") - - def addPortageSkip(self, test, info): - self.portage_skipped.append((test, info)) - if self.showAll: - self.stream.writeln("SKIP") - elif self.dots: - self.stream.write(".") - - def printErrors(self): - if self.dots or self.showAll: - self.stream.writeln() - self.printErrorList('ERROR', self.errors) - self.printErrorList('FAIL', self.failures) - self.printErrorList('TODO', self.todoed) - self.printErrorList('SKIP', self.portage_skipped) - -class TestCase(unittest.TestCase): - """ - We need a way to mark a unit test as "ok to fail" - This way someone can add a broken test and mark it as failed - and then fix the code later. This may not be a great approach - (broken code!!??!11oneone) but it does happen at times. - """ - - def __init__(self, *pargs, **kwargs): - unittest.TestCase.__init__(self, *pargs, **kwargs) - self.todo = False - self.portage_skip = None - self.cnf_path = cnf_path - self.cnf_etc_path = cnf_etc_path - self.bindir = cnf_bindir - self.sbindir = cnf_sbindir - - def defaultTestResult(self): - return TextTestResult() - - def run(self, result=None): - if result is None: result = self.defaultTestResult() - result.startTest(self) - testMethod = getattr(self, self._testMethodName) - try: - try: - self.setUp() - except SystemExit: - raise - except KeyboardInterrupt: - raise - except: - result.addError(self, sys.exc_info()) - return - - ok = False - try: - testMethod() - ok = True - except SkipTest as e: - result.addPortageSkip(self, "%s: SKIP: %s" % - (testMethod, str(e))) - except self.failureException: - if self.portage_skip is not None: - if self.portage_skip is True: - result.addPortageSkip(self, "%s: SKIP" % testMethod) - else: - result.addPortageSkip(self, "%s: SKIP: %s" % - (testMethod, self.portage_skip)) - elif self.todo: - result.addTodo(self, "%s: TODO" % testMethod) - else: - result.addFailure(self, sys.exc_info()) - except (KeyboardInterrupt, SystemExit): - raise - except: - result.addError(self, sys.exc_info()) - - try: - self.tearDown() - except SystemExit: - raise - except KeyboardInterrupt: - raise - except: - result.addError(self, sys.exc_info()) - ok = False - if ok: - result.addSuccess(self) - finally: - result.stopTest(self) - - def assertRaisesMsg(self, msg, excClass, callableObj, *args, **kwargs): - """Fail unless an exception of class excClass is thrown - by callableObj when invoked with arguments args and keyword - arguments kwargs. If a different type of exception is - thrown, it will not be caught, and the test case will be - deemed to have suffered an error, exactly as for an - unexpected exception. - """ - try: - callableObj(*args, **kwargs) - except excClass: - return - else: - if hasattr(excClass, '__name__'): excName = excClass.__name__ - else: excName = str(excClass) - raise self.failureException("%s not raised: %s" % (excName, msg)) - - def assertExists(self, path): - """Make sure |path| exists""" - if not os.path.exists(path): - msg = ['path is missing: %s' % (path,)] - while path != '/': - path = os.path.dirname(path) - if not path: - # If we're given something like "foo", abort once we get to "". - break - result = os.path.exists(path) - msg.append('\tos.path.exists(%s): %s' % (path, result)) - if result: - msg.append('\tcontents: %r' % os.listdir(path)) - break - raise self.failureException('\n'.join(msg)) - - def assertNotExists(self, path): - """Make sure |path| does not exist""" - if os.path.exists(path): - raise self.failureException('path exists when it should not: %s' % path) - -if unittest_skip_shims: - # Shim code for <python-2.7. - class SkipTest(Exception): - """unittest.SkipTest shim for <python-2.7""" - - def skipTest(self, reason): - raise SkipTest(reason) - setattr(TestCase, 'skipTest', skipTest) - - def assertIn(self, member, container, msg=None): - self.assertTrue(member in container, msg=msg) - setattr(TestCase, 'assertIn', assertIn) - - def assertNotIn(self, member, container, msg=None): - self.assertFalse(member in container, msg=msg) - setattr(TestCase, 'assertNotIn', assertNotIn) - -class TextTestRunner(unittest.TextTestRunner): - """ - We subclass unittest.TextTestRunner to output SKIP for tests that fail but are skippable - """ - - def _makeResult(self): - return TextTestResult(self.stream, self.descriptions, self.verbosity) - - def run(self, test): - """ - Run the given test case or test suite. - """ - result = self._makeResult() - startTime = time.time() - test(result) - stopTime = time.time() - timeTaken = stopTime - startTime - result.printErrors() - self.stream.writeln(result.separator2) - run = result.testsRun - self.stream.writeln("Ran %d test%s in %.3fs" % - (run, run != 1 and "s" or "", timeTaken)) - self.stream.writeln() - if not result.wasSuccessful(): - self.stream.write("FAILED (") - failed = len(result.failures) - errored = len(result.errors) - if failed: - self.stream.write("failures=%d" % failed) - if errored: - if failed: self.stream.write(", ") - self.stream.write("errors=%d" % errored) - self.stream.writeln(")") - else: - self.stream.writeln("OK") - return result - -test_cps = ['sys-apps/portage', 'virtual/portage'] -test_versions = ['1.0', '1.0-r1', '2.3_p4', '1.0_alpha57'] -test_slots = [None, '1', 'gentoo-sources-2.6.17', 'spankywashere'] -test_usedeps = ['foo', '-bar', ('foo', 'bar'), - ('foo', '-bar'), ('foo?', '!bar?')] diff --git a/repoman/pym/repoman/tests/__test__.py b/repoman/pym/repoman/tests/__test__.py deleted file mode 100644 index e69de29bb..000000000 --- a/repoman/pym/repoman/tests/__test__.py +++ /dev/null diff --git a/repoman/pym/repoman/tests/changelog/__init__.py b/repoman/pym/repoman/tests/changelog/__init__.py deleted file mode 100644 index 532918b6a..000000000 --- a/repoman/pym/repoman/tests/changelog/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright 2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 diff --git a/repoman/pym/repoman/tests/changelog/__test__.py b/repoman/pym/repoman/tests/changelog/__test__.py deleted file mode 100644 index e69de29bb..000000000 --- a/repoman/pym/repoman/tests/changelog/__test__.py +++ /dev/null diff --git a/repoman/pym/repoman/tests/changelog/test_echangelog.py b/repoman/pym/repoman/tests/changelog/test_echangelog.py deleted file mode 100644 index 1640be268..000000000 --- a/repoman/pym/repoman/tests/changelog/test_echangelog.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright 2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import tempfile -import time - -from portage import os -from portage import shutil -from portage.tests import TestCase -from repoman.utilities import UpdateChangeLog - -class RepomanEchangelogTestCase(TestCase): - - def setUp(self): - super(RepomanEchangelogTestCase, self).setUp() - - self.tmpdir = tempfile.mkdtemp(prefix='repoman.echangelog.') - - self.skel_changelog = os.path.join(self.tmpdir, 'skel.ChangeLog') - skel = [ - '# ChangeLog for <CATEGORY>/<PACKAGE_NAME>\n', - '# Copyright 1999-2000 Gentoo Foundation; Distributed under the GPL v2\n', - '# $Header: $\n' - ] - self._writelines(self.skel_changelog, skel) - - self.cat = 'mycat' - self.pkg = 'mypkg' - self.pkgdir = os.path.join(self.tmpdir, self.cat, self.pkg) - os.makedirs(self.pkgdir) - - self.header_pkg = '# ChangeLog for %s/%s\n' % (self.cat, self.pkg) - self.header_copyright = '# Copyright 1999-%s Gentoo Foundation; Distributed under the GPL v2\n' % \ - time.strftime('%Y', time.gmtime()) - self.header_cvs = '# $Header: $\n' - - self.changelog = os.path.join(self.pkgdir, 'ChangeLog') - - self.user = 'Testing User <portage@gentoo.org>' - - def tearDown(self): - super(RepomanEchangelogTestCase, self).tearDown() - shutil.rmtree(self.tmpdir) - - def _readlines(self, file): - with open(file, 'r') as f: - return f.readlines() - - def _writelines(self, file, data): - with open(file, 'w') as f: - f.writelines(data) - - def testRejectRootUser(self): - self.assertEqual(UpdateChangeLog(self.pkgdir, 'me <root@gentoo.org>', '', '', '', '', quiet=True), None) - - def testMissingSkelFile(self): - # Test missing ChangeLog, but with empty skel (i.e. do nothing). - UpdateChangeLog(self.pkgdir, self.user, 'test!', '/does/not/exist', self.cat, self.pkg, quiet=True) - actual_cl = self._readlines(self.changelog) - self.assertTrue(len(actual_cl[0]) > 0) - - def testEmptyChangeLog(self): - # Make sure we do the right thing with a 0-byte ChangeLog - open(self.changelog, 'w').close() - UpdateChangeLog(self.pkgdir, self.user, 'test!', self.skel_changelog, self.cat, self.pkg, quiet=True) - actual_cl = self._readlines(self.changelog) - self.assertEqual(actual_cl[0], self.header_pkg) - self.assertEqual(actual_cl[1], self.header_copyright) - self.assertEqual(actual_cl[2], self.header_cvs) - - def testCopyrightUpdate(self): - # Make sure updating the copyright line works - UpdateChangeLog(self.pkgdir, self.user, 'test!', self.skel_changelog, self.cat, self.pkg, quiet=True) - actual_cl = self._readlines(self.changelog) - self.assertEqual(actual_cl[1], self.header_copyright) - - def testSkelHeader(self): - # Test skel.ChangeLog -> ChangeLog - UpdateChangeLog(self.pkgdir, self.user, 'test!', self.skel_changelog, self.cat, self.pkg, quiet=True) - actual_cl = self._readlines(self.changelog) - self.assertEqual(actual_cl[0], self.header_pkg) - self.assertNotEqual(actual_cl[-1], '\n') - - def testExistingGoodHeader(self): - # Test existing ChangeLog (correct values) - self._writelines(self.changelog, [self.header_pkg]) - - UpdateChangeLog(self.pkgdir, self.user, 'test!', self.skel_changelog, self.cat, self.pkg, quiet=True) - actual_cl = self._readlines(self.changelog) - self.assertEqual(actual_cl[0], self.header_pkg) - - def testExistingBadHeader(self): - # Test existing ChangeLog (wrong values) - self._writelines(self.changelog, ['# ChangeLog for \n']) - - UpdateChangeLog(self.pkgdir, self.user, 'test!', self.skel_changelog, self.cat, self.pkg, quiet=True) - actual_cl = self._readlines(self.changelog) - self.assertEqual(actual_cl[0], self.header_pkg) - - def testTrailingNewlines(self): - # Make sure trailing newlines get chomped. - self._writelines(self.changelog, ['#\n', 'foo\n', '\n', 'bar\n', '\n', '\n']) - - UpdateChangeLog(self.pkgdir, self.user, 'test!', self.skel_changelog, self.cat, self.pkg, quiet=True) - actual_cl = self._readlines(self.changelog) - self.assertNotEqual(actual_cl[-1], '\n') diff --git a/repoman/pym/repoman/tests/commit/__init__.py b/repoman/pym/repoman/tests/commit/__init__.py deleted file mode 100644 index d74fd94a7..000000000 --- a/repoman/pym/repoman/tests/commit/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright 2011-2018 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 diff --git a/repoman/pym/repoman/tests/commit/__test__.py b/repoman/pym/repoman/tests/commit/__test__.py deleted file mode 100644 index 8b1378917..000000000 --- a/repoman/pym/repoman/tests/commit/__test__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/repoman/pym/repoman/tests/commit/test_commitmsg.py b/repoman/pym/repoman/tests/commit/test_commitmsg.py deleted file mode 100644 index a734c1065..000000000 --- a/repoman/pym/repoman/tests/commit/test_commitmsg.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright 2011-2018 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from repoman.actions import Actions -from repoman.tests import TestCase - - -class CommitMessageVerificationTest(TestCase): - def assertGood(self, commitmsg): - res, expl = Actions.verify_commit_message(commitmsg) - self.assertTrue(res, - '''Commit message verification failed for: -%s - -Error: -%s''' % (commitmsg, expl)) - - def assertBad(self, commitmsg, reason_re): - res, expl = Actions.verify_commit_message(commitmsg) - self.assertFalse(res, - '''Commit message verification succeeded unexpectedly, for: -%s - -Expected: /%s/''' % (commitmsg, reason_re)) - self.assertNotIn('\n', expl.strip(), - '''Commit message verification returned multiple errors (one expected): -%s - -Expected: /%s/ -Errors: -%s''' % (commitmsg, reason_re, expl)) - self.assertRegexpMatches(expl, reason_re, - '''Commit message verification did not return expected error, for: -%s - -Expected: /%s/ -Errors: -%s''' % (commitmsg, reason_re, expl)) - - def test_summary_only(self): - self.assertGood('dev-foo/bar: Actually good commit message') - - def test_summary_and_body(self): - self.assertGood('''dev-foo/bar: Good commit message - -Extended description goes here and is properly wrapped at 72 characters -which is very nice and blah blah. - -Another paragraph for the sake of having one.''') - - def test_summary_and_footer(self): - self.assertGood('''dev-foo/bar: Good commit message - -Closes: https://bugs.gentoo.org/NNNNNN''') - - def test_summary_body_and_footer(self): - self.assertGood('''dev-foo/bar: Good commit message - -Extended description goes here and is properly wrapped at 72 characters -which is very nice and blah blah. - -Another paragraph for the sake of having one. - -Closes: https://bugs.gentoo.org/NNNNNN''') - - def test_summary_without_unit_name(self): - self.assertBad('Version bump', r'summary.*logical unit name') - - def test_multiline_summary(self): - self.assertBad('''dev-foo/bar: Commit message with very long summary -that got wrapped because of length''', r'single.*line.*summary') - - def test_overlong_summary(self): - self.assertBad('dev-foo/bar: Commit message with very long summary \ -in a single line that should trigger an explicit error', - r'summary.*too long') - - def test_summary_with_very_long_package_name(self): - self.assertGood('dev-foo/foo-bar-bar-baz-bar-bar-foo-bar-bar-\ -baz-foo-baz-baz-foo: We do not fail because pkgname was long') - - def test_multiple_footers(self): - self.assertBad('''dev-foo/bar: Good summary - -Bug: https://bugs.gentoo.org/NNNNNN - -Closes: https://github.com/gentoo/gentoo/pull/NNNN''', r'multiple footer') - - def test_gentoo_bug(self): - self.assertBad('''dev-foo/bar: Good summary - -Gentoo-Bug: NNNNNN''', r'Gentoo-Bug') - - def test_bug_with_number(self): - self.assertBad('''dev-foo/bar: Good summary - -Bug: NNNNNN''', r'Bug.*full URL') - - def test_closes_with_number(self): - self.assertBad('''dev-foo/bar: Good summary - -Closes: NNNNNN''', r'Closes.*full URL') - - def test_body_too_long(self): - self.assertBad('''dev-foo/bar: Good summary - -But the body is not wrapped properly and has very long lines that are \ -very hard to read and blah blah blah -blah blah.''', r'body.*wrapped') diff --git a/repoman/pym/repoman/tests/runTests.py b/repoman/pym/repoman/tests/runTests.py deleted file mode 100644 index ed73592a1..000000000 --- a/repoman/pym/repoman/tests/runTests.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env python -# runTests.py -- Portage Unit Test Functionality -# Copyright 2006-2017 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import os -import sys -import os.path as osp -import grp -import platform -import pwd -import signal - - -def debug_signal(signum, frame): - import pdb - pdb.set_trace() - -if platform.python_implementation() == 'Jython': - debug_signum = signal.SIGUSR2 # bug #424259 -else: - debug_signum = signal.SIGUSR1 - -signal.signal(debug_signum, debug_signal) - -# Pretend that the current user's uid/gid are the 'portage' uid/gid, -# so things go smoothly regardless of the current user and global -# user/group configuration. -os.environ["PORTAGE_USERNAME"] = pwd.getpwuid(os.getuid()).pw_name -os.environ["PORTAGE_GRPNAME"] = grp.getgrgid(os.getgid()).gr_name - -# Insert our parent dir so we can do shiny import "tests" -# This line courtesy of Marienz and Pkgcore ;) -repoman_pym = osp.dirname(osp.dirname(osp.dirname(osp.realpath(__file__)))) -sys.path.insert(0, repoman_pym) - -# Add in the parent portage python modules -portage_pym = osp.dirname(osp.dirname(repoman_pym)) + '/pym' -sys.path.insert(0, portage_pym) - -# import our centrally initialized portage instance -from repoman._portage import portage -portage._internal_caller = True - -# Ensure that we don't instantiate portage.settings, so that tests should -# work the same regardless of global configuration file state/existence. -portage._disable_legacy_globals() -from portage.util._eventloop.global_event_loop import global_event_loop - -if os.environ.get('NOCOLOR') in ('yes', 'true'): - portage.output.nocolor() - -import repoman.tests as tests -from portage.const import PORTAGE_BIN_PATH -path = os.environ.get("PATH", "").split(":") -path = [x for x in path if x] - -insert_bin_path = True -try: - insert_bin_path = not path or \ - not os.path.samefile(path[0], PORTAGE_BIN_PATH) -except OSError: - pass - -if insert_bin_path: - path.insert(0, PORTAGE_BIN_PATH) - os.environ["PATH"] = ":".join(path) - -if __name__ == "__main__": - try: - sys.exit(tests.main()) - finally: - global_event_loop().close() diff --git a/repoman/pym/repoman/tests/simple/__init__.py b/repoman/pym/repoman/tests/simple/__init__.py deleted file mode 100644 index 532918b6a..000000000 --- a/repoman/pym/repoman/tests/simple/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright 2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 diff --git a/repoman/pym/repoman/tests/simple/__test__.py b/repoman/pym/repoman/tests/simple/__test__.py deleted file mode 100644 index 8b1378917..000000000 --- a/repoman/pym/repoman/tests/simple/__test__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/repoman/pym/repoman/tests/simple/test_simple.py b/repoman/pym/repoman/tests/simple/test_simple.py deleted file mode 100644 index 3d7a70ad0..000000000 --- a/repoman/pym/repoman/tests/simple/test_simple.py +++ /dev/null @@ -1,309 +0,0 @@ -# Copyright 2011-2018 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import subprocess -import sys -import time - -from repoman._portage import portage -from portage import os -from portage import _unicode_decode -from portage.process import find_binary -from portage.tests.resolver.ResolverPlayground import ResolverPlayground -from portage.util import ensure_dirs -from repoman import REPOMAN_BASE_PATH -from repoman.copyrights import update_copyright_year -from repoman.tests import TestCase - - -class SimpleRepomanTestCase(TestCase): - - def testCopyrightUpdate(self): - test_cases = ( - ( - '2011', - '# Copyright 1999-2008 Gentoo Foundation; Distributed under the GPL v2', - '# Copyright 1999-2011 Gentoo Foundation; Distributed under the GPL v2', - ), - ( - '2011', - '# Copyright 1999 Gentoo Foundation; Distributed under the GPL v2', - '# Copyright 1999-2011 Gentoo Foundation; Distributed under the GPL v2', - ), - ( - '1999', - '# Copyright 1999 Gentoo Foundation; Distributed under the GPL v2', - '# Copyright 1999 Gentoo Foundation; Distributed under the GPL v2', - ), - ) - - for year, before, after in test_cases: - self.assertEqual(update_copyright_year(year, before), after) - - def _must_skip(self): - xmllint = find_binary("xmllint") - if not xmllint: - return "xmllint not found" - - try: - __import__("xml.etree.ElementTree") - __import__("xml.parsers.expat").parsers.expat.ExpatError - except (AttributeError, ImportError): - return "python is missing xml support" - - def testSimple(self): - debug = False - - skip_reason = self._must_skip() - if skip_reason: - self.portage_skip = skip_reason - self.assertFalse(True, skip_reason) - return - - copyright_header = """# Copyright 1999-%s Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 -# $Header: $ -""" % time.gmtime().tm_year - - repo_configs = { - "test_repo": { - "layout.conf": - ( - "update-changelog = true", - ), - } - } - - profiles = ( - ("x86", "default/linux/x86/test_profile", "stable"), - ("x86", "default/linux/x86/test_dev", "dev"), - ("x86", "default/linux/x86/test_exp", "exp"), - ) - - profile = { - "eapi": ("5",), - "package.use.stable.mask": ("dev-libs/A flag",) - } - - ebuilds = { - "dev-libs/A-0": { - "COPYRIGHT_HEADER" : copyright_header, - "DESCRIPTION" : "Desc goes here", - "EAPI" : "5", - "HOMEPAGE" : "https://example.com", - "IUSE" : "flag", - "KEYWORDS": "x86", - "LICENSE": "GPL-2", - "RDEPEND": "flag? ( dev-libs/B[flag] )", - }, - "dev-libs/A-1": { - "COPYRIGHT_HEADER" : copyright_header, - "DESCRIPTION" : "Desc goes here", - "EAPI" : "4", - "HOMEPAGE" : "https://example.com", - "IUSE" : "flag", - "KEYWORDS": "~x86", - "LICENSE": "GPL-2", - "RDEPEND": "flag? ( dev-libs/B[flag] )", - }, - "dev-libs/B-1": { - "COPYRIGHT_HEADER" : copyright_header, - "DESCRIPTION" : "Desc goes here", - "EAPI" : "4", - "HOMEPAGE" : "https://example.com", - "IUSE" : "flag", - "KEYWORDS": "~x86", - "LICENSE": "GPL-2", - }, - "dev-libs/C-0": { - "COPYRIGHT_HEADER" : copyright_header, - "DESCRIPTION" : "Desc goes here", - "EAPI" : "4", - "HOMEPAGE" : "https://example.com", - "IUSE" : "flag", - # must be unstable, since dev-libs/A[flag] is stable masked - "KEYWORDS": "~x86", - "LICENSE": "GPL-2", - "RDEPEND": "flag? ( dev-libs/A[flag] )", - }, - } - licenses = ["GPL-2"] - arch_list = ["x86"] - metadata_xsd = os.path.join(REPOMAN_BASE_PATH, "cnf/metadata.xsd") - metadata_xml_files = ( - ( - "dev-libs/A", - { - "flags" : "<flag name='flag' restrict='>=dev-libs/A-0'>Description of how USE='flag' affects this package</flag>", - }, - ), - ( - "dev-libs/B", - { - "flags" : "<flag name='flag'>Description of how USE='flag' affects this package</flag>", - }, - ), - ( - "dev-libs/C", - { - "flags" : "<flag name='flag'>Description of how USE='flag' affects this package</flag>", - }, - ), - ) - - use_desc = ( - ("flag", "Description of how USE='flag' affects packages"), - ) - - playground = ResolverPlayground(ebuilds=ebuilds, - profile=profile, repo_configs=repo_configs, debug=debug) - settings = playground.settings - eprefix = settings["EPREFIX"] - eroot = settings["EROOT"] - portdb = playground.trees[playground.eroot]["porttree"].dbapi - homedir = os.path.join(eroot, "home") - distdir = os.path.join(eprefix, "distdir") - test_repo_location = settings.repositories["test_repo"].location - profiles_dir = os.path.join(test_repo_location, "profiles") - license_dir = os.path.join(test_repo_location, "licenses") - - repoman_cmd = (portage._python_interpreter, "-b", "-Wd", - os.path.join(self.bindir, "repoman")) - - git_binary = find_binary("git") - git_cmd = (git_binary,) - - cp_binary = find_binary("cp") - self.assertEqual(cp_binary is None, False, - "cp command not found") - cp_cmd = (cp_binary,) - - test_ebuild = portdb.findname("dev-libs/A-1") - self.assertFalse(test_ebuild is None) - - committer_name = "Gentoo Dev" - committer_email = "gentoo-dev@gentoo.org" - - git_test = ( - ("", repoman_cmd + ("manifest",)), - ("", git_cmd + ("config", "--global", "user.name", committer_name,)), - ("", git_cmd + ("config", "--global", "user.email", committer_email,)), - ("", git_cmd + ("init-db",)), - ("", git_cmd + ("add", ".")), - ("", git_cmd + ("commit", "-a", "-m", "add whole repo")), - ("", repoman_cmd + ("full", "-d")), - ("", cp_cmd + (test_ebuild, test_ebuild[:-8] + "2.ebuild")), - ("", git_cmd + ("add", test_ebuild[:-8] + "2.ebuild")), - ("", repoman_cmd + ("commit", "-m", "cat/pkg: bump to version 2")), - ("", cp_cmd + (test_ebuild, test_ebuild[:-8] + "3.ebuild")), - ("", git_cmd + ("add", test_ebuild[:-8] + "3.ebuild")), - ("dev-libs", repoman_cmd + ("commit", "-m", "cat/pkg: bump to version 3")), - ("", cp_cmd + (test_ebuild, test_ebuild[:-8] + "4.ebuild")), - ("", git_cmd + ("add", test_ebuild[:-8] + "4.ebuild")), - ("dev-libs/A", repoman_cmd + ("commit", "-m", "cat/pkg: bump to version 4")), - ) - - env = { - "PORTAGE_OVERRIDE_EPREFIX" : eprefix, - "DISTDIR" : distdir, - "GENTOO_COMMITTER_NAME" : committer_name, - "GENTOO_COMMITTER_EMAIL" : committer_email, - "HOME" : homedir, - "PATH" : os.environ["PATH"], - "PORTAGE_GRPNAME" : os.environ["PORTAGE_GRPNAME"], - "PORTAGE_USERNAME" : os.environ["PORTAGE_USERNAME"], - "PORTAGE_REPOSITORIES" : settings.repositories.config_string(), - "PYTHONDONTWRITEBYTECODE" : os.environ.get("PYTHONDONTWRITEBYTECODE", ""), - } - - if os.environ.get("SANDBOX_ON") == "1": - # avoid problems from nested sandbox instances - env["FEATURES"] = "-sandbox -usersandbox" - - dirs = [homedir, license_dir, profiles_dir, distdir] - try: - for d in dirs: - ensure_dirs(d) - with open(os.path.join(test_repo_location, "skel.ChangeLog"), 'w') as f: - f.write(copyright_header) - with open(os.path.join(profiles_dir, "profiles.desc"), 'w') as f: - for x in profiles: - f.write("%s %s %s\n" % x) - - # ResolverPlayground only created the first profile, - # so create the remaining ones. - for x in profiles[1:]: - sub_profile_dir = os.path.join(profiles_dir, x[1]) - ensure_dirs(sub_profile_dir) - for config_file, lines in profile.items(): - file_name = os.path.join(sub_profile_dir, config_file) - with open(file_name, "w") as f: - for line in lines: - f.write("%s\n" % line) - - for x in licenses: - open(os.path.join(license_dir, x), 'wb').close() - with open(os.path.join(profiles_dir, "arch.list"), 'w') as f: - for x in arch_list: - f.write("%s\n" % x) - with open(os.path.join(profiles_dir, "use.desc"), 'w') as f: - for k, v in use_desc: - f.write("%s - %s\n" % (k, v)) - for cp, xml_data in metadata_xml_files: - with open(os.path.join(test_repo_location, cp, "metadata.xml"), 'w') as f: - f.write(playground.metadata_xml_template % xml_data) - # Use a symlink to test_repo, in order to trigger bugs - # involving canonical vs. non-canonical paths. - test_repo_symlink = os.path.join(eroot, "test_repo_symlink") - os.symlink(test_repo_location, test_repo_symlink) - metadata_xsd_dest = os.path.join(test_repo_location, 'metadata/xml-schema/metadata.xsd') - os.makedirs(os.path.dirname(metadata_xsd_dest)) - os.symlink(metadata_xsd, metadata_xsd_dest) - - if debug: - # The subprocess inherits both stdout and stderr, for - # debugging purposes. - stdout = None - else: - # The subprocess inherits stderr so that any warnings - # triggered by python -Wd will be visible. - stdout = subprocess.PIPE - - for cwd in ("", "dev-libs", "dev-libs/A", "dev-libs/B"): - abs_cwd = os.path.join(test_repo_symlink, cwd) - proc = subprocess.Popen(repoman_cmd + ("full",), - cwd=abs_cwd, env=env, stdout=stdout) - - if debug: - proc.wait() - else: - output = proc.stdout.readlines() - proc.wait() - proc.stdout.close() - if proc.returncode != os.EX_OK: - for line in output: - sys.stderr.write(_unicode_decode(line)) - - self.assertEqual(os.EX_OK, proc.returncode, - "repoman failed in %s" % (cwd,)) - - if git_binary is not None: - for cwd, cmd in git_test: - abs_cwd = os.path.join(test_repo_symlink, cwd) - proc = subprocess.Popen(cmd, - cwd=abs_cwd, env=env, stdout=stdout) - - if debug: - proc.wait() - else: - output = proc.stdout.readlines() - proc.wait() - proc.stdout.close() - if proc.returncode != os.EX_OK: - for line in output: - sys.stderr.write(_unicode_decode(line)) - - self.assertEqual(os.EX_OK, proc.returncode, - "%s failed in %s" % (cmd, cwd,)) - finally: - playground.cleanup() diff --git a/repoman/pym/repoman/utilities.py b/repoman/pym/repoman/utilities.py deleted file mode 100644 index 1272f3fb6..000000000 --- a/repoman/pym/repoman/utilities.py +++ /dev/null @@ -1,570 +0,0 @@ -# -*- coding:utf-8 -*- -# repoman: Utilities -# Copyright 2007-2018 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -"""This module contains utility functions to help repoman find ebuilds to -scan""" - -from __future__ import print_function, unicode_literals - -__all__ = [ - "editor_is_executable", - "FindPackagesToScan", - "FindPortdir", - "get_commit_message_with_editor", - "get_committer_name", - "have_ebuild_dir", - "have_profile_dir", - "UpdateChangeLog" -] - -import errno -import io -from itertools import chain -import logging -import pwd -import stat -import sys -import time -import textwrap -import difflib -import tempfile - -# import our initialized portage instance -from repoman._portage import portage - -from portage import os -from portage import shutil -from portage import _encodings -from portage import _unicode_decode -from portage import _unicode_encode -from portage import util -from portage.localization import _ -from portage.process import find_binary -from portage.output import green - -from repoman.copyrights import update_copyright, update_copyright_year - - -normalize_path = util.normalize_path -util.initialize_logger() - -if sys.hexversion >= 0x3000000: - basestring = str - - -def have_profile_dir(path, maxdepth=3, filename="profiles.desc"): - """ - Try to figure out if 'path' has a profiles/ - dir in it by checking for the given filename. - """ - while path != "/" and maxdepth: - if os.path.exists(os.path.join(path, "profiles", filename)): - return normalize_path(path) - path = normalize_path(path + "/..") - maxdepth -= 1 - - -def have_ebuild_dir(path, maxdepth=3): - """ - Try to figure out if 'path' or a subdirectory contains one or more - ebuild files named appropriately for their parent directory. - """ - stack = [(normalize_path(path), 1)] - while stack: - path, depth = stack.pop() - basename = os.path.basename(path) - try: - listdir = os.listdir(path) - except OSError: - continue - for filename in listdir: - abs_filename = os.path.join(path, filename) - try: - st = os.stat(abs_filename) - except OSError: - continue - if stat.S_ISDIR(st.st_mode): - if depth < maxdepth: - stack.append((abs_filename, depth + 1)) - elif stat.S_ISREG(st.st_mode): - if filename.endswith(".ebuild") and \ - filename.startswith(basename + "-"): - return os.path.dirname(os.path.dirname(path)) - - -def FindPackagesToScan(settings, startdir, reposplit): - """ Try to find packages that need to be scanned - - Args: - settings - portage.config instance, preferably repoman_settings - startdir - directory that repoman was run in - reposplit - root of the repository - Returns: - A list of directories to scan - """ - - def AddPackagesInDir(path): - """ Given a list of dirs, add any packages in it """ - ret = [] - pkgdirs = os.listdir(path) - for d in pkgdirs: - if d == 'CVS' or d.startswith('.'): - continue - p = os.path.join(path, d) - - if os.path.isdir(p): - cat_pkg_dir = os.path.join(*p.split(os.path.sep)[-2:]) - logging.debug('adding %s to scanlist' % cat_pkg_dir) - ret.append(cat_pkg_dir) - return ret - - scanlist = [] - repolevel = len(reposplit) - if repolevel == 1: # root of the tree, startdir = repodir - for cat in settings.categories: - path = os.path.join(startdir, cat) - if not os.path.isdir(path): - continue - scanlist.extend(AddPackagesInDir(path)) - elif repolevel == 2: # category level, startdir = catdir - # We only want 1 segment of the directory, - # this is why we use catdir instead of startdir. - catdir = reposplit[-2] - if catdir not in settings.categories: - logging.warn( - '%s is not a valid category according to profiles/categories, ' - 'skipping checks in %s' % (catdir, catdir)) - else: - scanlist = AddPackagesInDir(catdir) - elif repolevel == 3: # pkgdir level, startdir = pkgdir - catdir = reposplit[-2] - pkgdir = reposplit[-1] - if catdir not in settings.categories: - logging.warn( - '%s is not a valid category according to profiles/categories, ' - 'skipping checks in %s' % (catdir, catdir)) - else: - path = os.path.join(catdir, pkgdir) - logging.debug('adding %s to scanlist' % path) - scanlist.append(path) - return scanlist - - -def editor_is_executable(editor): - """ - Given an EDITOR string, validate that it refers to - an executable. This uses shlex_split() to split the - first component and do a PATH lookup if necessary. - - @param editor: An EDITOR value from the environment. - @type: string - @rtype: bool - @return: True if an executable is found, False otherwise. - """ - editor_split = util.shlex_split(editor) - if not editor_split: - return False - filename = editor_split[0] - if not os.path.isabs(filename): - return find_binary(filename) is not None - return os.access(filename, os.X_OK) and os.path.isfile(filename) - - -def get_commit_message_with_editor(editor, message=None, prefix=""): - """ - Execute editor with a temporary file as it's argument - and return the file content afterwards. - - @param editor: An EDITOR value from the environment - @type: string - @param message: An iterable of lines to show in the editor. - @type: iterable - @param prefix: Suggested prefix for the commit message summary line. - @type: string - @rtype: string or None - @return: A string on success or None if an error occurs. - """ - commitmessagedir = tempfile.mkdtemp(".repoman.msg") - filename = os.path.join(commitmessagedir, "COMMIT_EDITMSG") - try: - with open(filename, "wb") as mymsg: - mymsg.write( - _unicode_encode(_( - prefix + - "\n\n# Please enter the commit message " - "for your changes.\n# (Comment lines starting " - "with '#' will not be included)\n"), - encoding=_encodings['content'], errors='backslashreplace')) - if message: - mymsg.write(b"#\n") - for line in message: - mymsg.write( - _unicode_encode( - "#" + line, encoding=_encodings['content'], - errors='backslashreplace')) - retval = os.system(editor + " '%s'" % filename) - if not (os.WIFEXITED(retval) and os.WEXITSTATUS(retval) == os.EX_OK): - return None - try: - with io.open(_unicode_encode( - filename, encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], errors='replace') as f: - mylines = f.readlines() - except OSError as e: - if e.errno != errno.ENOENT: - raise - del e - return None - return "".join(line for line in mylines if not line.startswith("#")) - finally: - try: - shutil.rmtree(commitmessagedir) - except OSError: - pass - - -def FindPortdir(settings): - """ Try to figure out what repo we are in and whether we are in a regular - tree or an overlay. - - Basic logic is: - - 1. Determine what directory we are in (supports symlinks). - 2. Build a list of directories from / to our current location - 3. Iterate over PORTDIR_OVERLAY, if we find a match, - search for a profiles directory in the overlay. If it has one, - make it portdir, otherwise make it portdir_overlay. - 4. If we didn't find an overlay in PORTDIR_OVERLAY, - see if we are in PORTDIR; if so, set portdir_overlay to PORTDIR. - If we aren't in PORTDIR, see if PWD has a profiles dir, if so, - set portdir_overlay and portdir to PWD, else make them False. - 5. If we haven't found portdir_overlay yet, - it means the user is doing something odd, report an error. - 6. If we haven't found a portdir yet, set portdir to PORTDIR. - - Args: - settings - portage.config instance, preferably repoman_settings - Returns: - list(portdir, portdir_overlay, location) - """ - - portdir = None - portdir_overlay = None - location = os.getcwd() - pwd = _unicode_decode(os.environ.get('PWD', ''), encoding=_encodings['fs']) - if pwd and pwd != location and os.path.realpath(pwd) == location: - # getcwd() returns the canonical path but that makes it hard for repoman to - # orient itself if the user has symlinks in their portage tree structure. - # We use os.environ["PWD"], if available, to get the non-canonical path of - # the current working directory (from the shell). - location = pwd - - location = normalize_path(location) - - path_ids = {} - p = location - s = None - while True: - s = os.stat(p) - path_ids[(s.st_dev, s.st_ino)] = p - if p == "/": - break - p = os.path.dirname(p) - if location[-1] != "/": - location += "/" - - for overlay in portage.util.shlex_split(settings["PORTDIR_OVERLAY"]): - overlay = os.path.realpath(overlay) - try: - s = os.stat(overlay) - except OSError: - continue - overlay = path_ids.get((s.st_dev, s.st_ino)) - if overlay is None: - continue - if overlay[-1] != "/": - overlay += "/" - if True: - portdir_overlay = overlay - subdir = location[len(overlay):] - if subdir and subdir[-1] != "/": - subdir += "/" - if have_profile_dir(location, subdir.count("/")): - portdir = portdir_overlay - break - - # Couldn't match location with anything from PORTDIR_OVERLAY, - # so fall back to have_profile_dir() checks alone. Assume that - # an overlay will contain at least a "repo_name" file while a - # master repo (portdir) will contain at least a "profiles.desc" - # file. - if not portdir_overlay: - portdir_overlay = have_profile_dir(location, filename="repo_name") - if not portdir_overlay: - portdir_overlay = have_ebuild_dir(location) - if portdir_overlay: - subdir = location[len(portdir_overlay):] - if subdir and subdir[-1] != os.sep: - subdir += os.sep - if have_profile_dir(location, subdir.count(os.sep)): - portdir = portdir_overlay - - if not portdir_overlay: - if (settings["PORTDIR"] + os.path.sep).startswith(location): - portdir_overlay = settings["PORTDIR"] - else: - portdir_overlay = have_profile_dir(location) - portdir = portdir_overlay - - if not portdir_overlay: - msg = 'Repoman is unable to determine PORTDIR or PORTDIR_OVERLAY' + \ - ' from the current working directory' - logging.critical(msg) - return (None, None, None) - - if not portdir: - portdir = settings["PORTDIR"] - - if not portdir_overlay.endswith('/'): - portdir_overlay += '/' - - if not portdir.endswith('/'): - portdir += '/' - - return [normalize_path(x) for x in (portdir, portdir_overlay, location)] - - -def get_committer_name(env=None): - """Generate a committer string like echangelog does.""" - if env is None: - env = os.environ - if 'GENTOO_COMMITTER_NAME' in env and 'GENTOO_COMMITTER_EMAIL' in env: - user = '%s <%s>' % ( - env['GENTOO_COMMITTER_NAME'], - env['GENTOO_COMMITTER_EMAIL']) - elif 'GENTOO_AUTHOR_NAME' in env and 'GENTOO_AUTHOR_EMAIL' in env: - user = '%s <%s>' % ( - env['GENTOO_AUTHOR_NAME'], - env['GENTOO_AUTHOR_EMAIL']) - elif 'ECHANGELOG_USER' in env: - user = env['ECHANGELOG_USER'] - else: - pwd_struct = pwd.getpwuid(os.getuid()) - gecos = pwd_struct.pw_gecos.split(',')[0] # bug #80011 - user = '%s <%s@gentoo.org>' % (gecos, pwd_struct.pw_name) - return user - - -def UpdateChangeLog( - pkgdir, user, msg, skel_path, category, package, - new=(), removed=(), changed=(), pretend=False, quiet=False): - """ - Write an entry to an existing ChangeLog, or create a new one. - Updates copyright year on changed files, and updates the header of - ChangeLog with the contents of skel.ChangeLog. - """ - - if '<root@' in user: - if not quiet: - logging.critical('Please set ECHANGELOG_USER or run as non-root') - return None - - # ChangeLog times are in UTC - gmtime = time.gmtime() - year = time.strftime('%Y', gmtime) - date = time.strftime('%d %b %Y', gmtime) - - cl_path = os.path.join(pkgdir, 'ChangeLog') - clold_lines = [] - clnew_lines = [] - old_header_lines = [] - header_lines = [] - - clold_file = None - try: - clold_file = io.open(_unicode_encode( - cl_path, encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], errors='replace') - except EnvironmentError: - pass - - f, clnew_path = tempfile.mkstemp() - - # construct correct header first - try: - if clold_file is not None: - # retain header from old ChangeLog - first_line = True - for line in clold_file: - line_strip = line.strip() - if line_strip and line[:1] != "#": - clold_lines.append(line) - break - # always make sure cat/pkg is up-to-date in case we are - # moving packages around, or copied from another pkg, or ... - if first_line: - if line.startswith('# ChangeLog for'): - line = '# ChangeLog for %s/%s\n' % (category, package) - first_line = False - old_header_lines.append(line) - header_lines.append(update_copyright_year(year, line)) - if not line_strip: - break - - clskel_file = None - if not header_lines: - # delay opening this until we find we need a header - try: - clskel_file = io.open(_unicode_encode( - skel_path, encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], - errors='replace') - except EnvironmentError: - pass - - if clskel_file is not None: - # read skel.ChangeLog up to first empty line - for line in clskel_file: - line_strip = line.strip() - if not line_strip: - break - line = line.replace('<CATEGORY>', category) - line = line.replace('<PACKAGE_NAME>', package) - line = update_copyright_year(year, line) - header_lines.append(line) - header_lines.append('\n') - clskel_file.close() - - # write new ChangeLog entry - clnew_lines.extend(header_lines) - newebuild = False - for fn in new: - if not fn.endswith('.ebuild'): - continue - ebuild = fn.split(os.sep)[-1][0:-7] - clnew_lines.append('*%s (%s)\n' % (ebuild, date)) - newebuild = True - if newebuild: - clnew_lines.append('\n') - trivial_files = ('ChangeLog', 'Manifest') - display_new = [ - '+' + elem - for elem in new - if elem not in trivial_files] - display_removed = [ - '-' + elem - for elem in removed] - display_changed = [ - elem for elem in changed - if elem not in trivial_files] - if not (display_new or display_removed or display_changed): - # If there's nothing else to display, show one of the - # trivial files. - for fn in trivial_files: - if fn in new: - display_new = ['+' + fn] - break - elif fn in changed: - display_changed = [fn] - break - - display_new.sort() - display_removed.sort() - display_changed.sort() - - mesg = '%s; %s %s:' % (date, user, ', '.join(chain( - display_new, display_removed, display_changed))) - for line in textwrap.wrap( - mesg, 80, initial_indent=' ', subsequent_indent=' ', - break_on_hyphens=False): - clnew_lines.append('%s\n' % line) - for line in textwrap.wrap( - msg, 80, initial_indent=' ', subsequent_indent=' '): - clnew_lines.append('%s\n' % line) - # Don't append a trailing newline if the file is new. - if clold_file is not None: - clnew_lines.append('\n') - - f = io.open( - f, mode='w', encoding=_encodings['repo.content'], - errors='backslashreplace') - - for line in clnew_lines: - f.write(line) - - # append stuff from old ChangeLog - if clold_file is not None: - - if clold_lines: - # clold_lines may contain a saved non-header line - # that we want to write first. - # Also, append this line to clnew_lines so that the - # unified_diff call doesn't show it as removed. - for line in clold_lines: - f.write(line) - clnew_lines.append(line) - - else: - # ensure that there is no more than one blank - # line after our new entry - for line in clold_file: - if line.strip(): - f.write(line) - break - - # Now prepend old_header_lines to clold_lines, for use - # in the unified_diff call below. - clold_lines = old_header_lines + clold_lines - - # Trim any trailing newlines. - lines = clold_file.readlines() - clold_file.close() - while lines and lines[-1] == '\n': - del lines[-1] - f.writelines(lines) - f.close() - - # show diff - if not quiet: - for line in difflib.unified_diff( - clold_lines, clnew_lines, - fromfile=cl_path, tofile=cl_path, n=0): - util.writemsg_stdout(line, noiselevel=-1) - util.writemsg_stdout("\n", noiselevel=-1) - - if pretend: - # remove what we've done - os.remove(clnew_path) - else: - # rename to ChangeLog, and set permissions - try: - clold_stat = os.stat(cl_path) - except OSError: - clold_stat = None - - shutil.move(clnew_path, cl_path) - - if clold_stat is None: - util.apply_permissions(cl_path, mode=0o644) - else: - util.apply_stat_permissions(cl_path, clold_stat) - - if clold_file is None: - return True - else: - return False - except IOError as e: - err = 'Repoman is unable to create/write to Changelog.new file: %s' % (e,) - logging.critical(err) - # try to remove if possible - try: - os.remove(clnew_path) - except OSError: - pass - return None - - -def repoman_sez(msg): - print (green("RepoMan sez:"), msg) |