diff options
| author |   Fabian Groffen <grobian@gentoo.org> | 2020-08-02 13:17:47 +0200 |
|---|---|---|
| committer | Fabian Groffen <grobian@gentoo.org> | 2020-08-02 14:32:49 +0200 |
| commit | 0d9cd144937a2a4388cb299fbcd753257b085970 (patch) | |
| tree | d6fe48fc6fff0ded98af3110fb3061cf19d27620 | |
| parent | Merge remote-tracking branch 'overlays-gentoo-org/master' into prefix (diff) | |
| parent | Fix R0205 across all of repo. (diff) | |
| download | portage-0d9cd144.tar.gz portage-0d9cd144.tar.bz2 portage-0d9cd144.zip | |
Merge remote-tracking branch 'overlays-gentoo-org/master' into prefix
Signed-off-by: Fabian Groffen <grobian@gentoo.org>
328 files changed, 2638 insertions, 3095 deletions
diff --git a/.travis.yml b/.travis.yml index 6d3afa4ce..b9f6c85f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ dist: bionic language: python python: - - 2.7 - 3.6 - 3.7 - 3.8 @@ -1,5 +1,16 @@ News (mainly features/major bug fixes) +portage-3.0.0 +-------------- +* Dropped support for Python 2.x. + +portage-2.3.101 +-------------- +* The new PORTAGE_LOG_FILTER_FILE_CMD make.conf(5) variable specifies a + command that filters build log output to a log file. In order to + filter ANSI escape codes from build logs, ansifilter(1) is a + convenient setting for this variable. + portage-2.3.100 -------------- * New BINPKG_COMPRESS=zstd default (does not apply to installed systems @@ -19,7 +19,7 @@ other package managers. Dependencies ============ -Python and Bash should be the only hard dependencies. Python 2.7 is the +Python and Bash should be the only hard dependencies. Python 3.6 is the minimum supported version. Native Extensions diff --git a/RELEASE-NOTES b/RELEASE-NOTES index cf4a04c29..228da8e77 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -1,6 +1,49 @@ Release Notes; upgrade information mainly. Features/major bugfixes are listed in NEWS +portage-3.0.1 +================================== +* Bug Fixes: + - Bug 730192 Replace os.fork with multiprocessing.Process, and fix + regression in portage-3.0.0 involving eerror messages for fetch + failures + +portage-3.0.0 +================================== +* Bug Fixes: + - Bug 703698 Improve repos.conf handling of boolean settings + - Bug 721516 ecompress complains about compressed files that have + been excluded by docompress -x + - Bug 729852 Set XTerm titles for konsole + - Bug 731114 Drop support for python2.7 + - Bug 731246 man ebuild(5) has incorrect example of $P and $PN + - Bug 732378 Use lru_cache for use_reduce, vercmp, and catpkgsplit + - Bug 733154 List of ignored warnings in .desktop files should be + updated for compatibility with >=desktop-file-utils-0.25 + +portage-2.3.103 +================================== +* Bug Fixes: + - Bug 709746 set non-blocking for build_logger stdin in EbuildPhase + _elog method + - Bug 727522 ecompress: fix "Argument list too long" for sed + +portage-2.3.102 +================================== +* Bug Fixes: + - Bug 716636 Fix emerge hang triggered by unsafe remove_reader and + remove_writer calls related to bug 709746 + +portage-2.3.101 +================================== +* Bug Fixes: + - Bug 661518 repos.conf: Add bool sync-openpgp-key-refresh option + - Bug 709746 New PORTAGE_LOG_FILTER_FILE_CMD variable specifies a + command that filters build log output to a log file + - Bug 719810 Escape percent-signs in mirror url + - Bug 725934 _better_cache._scan_cat: Avoid stat calls + - Bug 728046 ecompress: Prefix eqawarn messages with QA Notice + portage-2.3.100 ================================== * Bug Fixes: diff --git a/bin/binhost-snapshot b/bin/binhost-snapshot index 41d556831..fbecfa8bb 100755 --- a/bin/binhost-snapshot +++ b/bin/binhost-snapshot @@ -8,10 +8,7 @@ import os import sys import textwrap -try: - from urllib.parse import urlparse -except ImportError: - from urlparse import urlparse +from urllib.parse import urlparse from os import path as osp if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), ".portage_not_installed")): diff --git a/bin/check-implicit-pointer-usage.py b/bin/check-implicit-pointer-usage.py index 7921d005d..208b3ac97 100755 --- a/bin/check-implicit-pointer-usage.py +++ b/bin/check-implicit-pointer-usage.py @@ -33,22 +33,10 @@ pointer_pattern = ( + r"|" + r"cast to pointer from integer of different size)") -if sys.hexversion < 0x3000000: - # Use encoded byte strings in python-2.x, since the python ebuilds are - # known to remove the encodings module when USE=build is enabled (thus - # disabling unicode decoding/encoding). The portage module has a - # workaround for this, but currently we don't import that here since we - # don't want to trigger potential sandbox violations due to stale pyc - # files for the portage module. - unicode_quote_open = '\xE2\x80\x98' - unicode_quote_close = '\xE2\x80\x99' - def write(msg): - sys.stdout.write(msg) -else: - unicode_quote_open = '\u2018' - unicode_quote_close = '\u2019' - def write(msg): - sys.stdout.buffer.write(msg.encode('utf_8', 'backslashreplace')) +unicode_quote_open = '\u2018' +unicode_quote_close = '\u2019' +def write(msg): + sys.stdout.buffer.write(msg.encode('utf_8', 'backslashreplace')) pointer_pattern = re.compile(pointer_pattern) @@ -57,10 +45,7 @@ last_implicit_linenum = -1 last_implicit_func = "" while True: - if sys.hexversion >= 0x3000000: - line = sys.stdin.buffer.readline().decode('utf_8', 'replace') - else: - line = sys.stdin.readline() + line = sys.stdin.buffer.readline().decode('utf_8', 'replace') if not line: break # translate unicode open/close quotes to ascii ones diff --git a/bin/chmod-lite.py b/bin/chmod-lite.py index b9a4fc9a2..d4dad8401 100755 --- a/bin/chmod-lite.py +++ b/bin/chmod-lite.py @@ -12,12 +12,11 @@ os.chdir(os.environ["__PORTAGE_HELPER_CWD"]) def main(files): - if sys.hexversion >= 0x3000000: - # We can't trust that the filesystem encoding (locale dependent) - # correctly matches the arguments, so use surrogateescape to - # pass through the original argv bytes for Python 3. - fs_encoding = sys.getfilesystemencoding() - files = [x.encode(fs_encoding, 'surrogateescape') for x in files] + # We can't trust that the filesystem encoding (locale dependent) + # correctly matches the arguments, so use surrogateescape to + # pass through the original argv bytes for Python 3. + fs_encoding = sys.getfilesystemencoding() + files = [x.encode(fs_encoding, 'surrogateescape') for x in files] for filename in files: # Emulate 'chmod -fR a+rX,u+w,g-w,o-w' with minimal chmod calls. diff --git a/bin/chpathtool.py b/bin/chpathtool.py index b9483b841..cd08ed58b 100755 --- a/bin/chpathtool.py +++ b/bin/chpathtool.py @@ -27,7 +27,7 @@ else: # magic module seems to be broken magic = None -class IsTextFile(object): +class IsTextFile: def __init__(self): if magic is not None: @@ -128,12 +128,8 @@ def chpath_inplace(filename, is_text_file, old, new): f.close() if modified: - if sys.hexversion >= 0x3030000: - orig_mtime = orig_stat.st_mtime_ns - os.utime(filename, ns=(orig_mtime, orig_mtime)) - else: - orig_mtime = orig_stat[stat.ST_MTIME] - os.utime(filename, (orig_mtime, orig_mtime)) + orig_mtime = orig_stat.st_mtime_ns + os.utime(filename, ns=(orig_mtime, orig_mtime)) return modified def chpath_inplace_symlink(filename, st, old, new): diff --git a/bin/dispatch-conf b/bin/dispatch-conf index c05215fc3..2a9db88a9 100755 --- a/bin/dispatch-conf +++ b/bin/dispatch-conf @@ -11,7 +11,7 @@ # dialog menus # -from __future__ import print_function, unicode_literals +from __future__ import print_function import atexit import io diff --git a/bin/dohtml.py b/bin/dohtml.py index f99cd9812..7be1241eb 100755 --- a/bin/dohtml.py +++ b/bin/dohtml.py @@ -28,7 +28,7 @@ # - will do as 'dohtml -r', but ignore directories named CVS, SCCS, RCS # -from __future__ import print_function, unicode_literals +from __future__ import print_function import os as _os import sys @@ -163,12 +163,11 @@ def print_help(): def parse_args(): argv = sys.argv[:] - if sys.hexversion >= 0x3000000: - # We can't trust that the filesystem encoding (locale dependent) - # correctly matches the arguments, so use surrogateescape to - # pass through the original argv bytes for Python 3. - fs_encoding = sys.getfilesystemencoding() - argv = [x.encode(fs_encoding, 'surrogateescape') for x in argv] + # We can't trust that the filesystem encoding (locale dependent) + # correctly matches the arguments, so use surrogateescape to + # pass through the original argv bytes for Python 3. + fs_encoding = sys.getfilesystemencoding() + argv = [x.encode(fs_encoding, 'surrogateescape') for x in argv] for x, arg in enumerate(argv): try: diff --git a/bin/doins.py b/bin/doins.py index 6bc30c90b..4929cb90a 100644 --- a/bin/doins.py +++ b/bin/doins.py @@ -110,10 +110,6 @@ def _parse_install_options( parser.add_argument('-p', '--preserve-timestamps', action='store_true') split_options = shlex.split(options) namespace, remaining = parser.parse_known_args(split_options) - if namespace.preserve_timestamps and sys.version_info < (3, 3): - # -p is not supported in this case, since timestamps cannot - # be preserved with full precision - remaining.append('-p') # Because parsing '--mode' option is partially supported. If unknown # arg for --mode is passed, namespace.mode is set to None. if remaining or namespace.mode is None: @@ -151,18 +147,10 @@ def _set_timestamps(source_stat, dest): source_stat: stat result for the source file. dest: path to the dest file. """ - os.utime(dest, (source_stat.st_atime, source_stat.st_mtime)) + os.utime(dest, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns)) -if sys.version_info >= (3, 3): - def _set_timestamps_ns(source_stat, dest): - os.utime(dest, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns)) - - _set_timestamps_ns.__doc__ = _set_timestamps.__doc__ - _set_timestamps = _set_timestamps_ns - - -class _InsInProcessInstallRunner(object): +class _InsInProcessInstallRunner: """Implements `install` command behavior running in a process.""" def __init__(self, opts, parsed_options): @@ -269,7 +257,7 @@ class _InsInProcessInstallRunner(object): return False -class _InsSubprocessInstallRunner(object): +class _InsSubprocessInstallRunner: """Runs `install` command in a subprocess to install a file.""" def __init__(self, split_options): @@ -295,7 +283,7 @@ class _InsSubprocessInstallRunner(object): return subprocess.call(command) == 0 -class _DirInProcessInstallRunner(object): +class _DirInProcessInstallRunner: """Implements `install` command behavior running in a process.""" def __init__(self, parsed_options): @@ -321,7 +309,7 @@ class _DirInProcessInstallRunner(object): _set_attributes(self._parsed_options, dest) -class _DirSubprocessInstallRunner(object): +class _DirSubprocessInstallRunner: """Runs `install` command to create a directory.""" def __init__(self, split_options): @@ -343,7 +331,7 @@ class _DirSubprocessInstallRunner(object): subprocess.check_call(command) -class _InstallRunner(object): +class _InstallRunner: """Handles `install` command operation. Runs operations which `install` command should work. If possible, @@ -514,10 +502,9 @@ def _parse_args(argv): # Encode back to the original byte stream. Please see # http://bugs.python.org/issue8776. - if sys.version_info.major >= 3: - opts.distdir = os.fsencode(opts.distdir) + b'/' - opts.dest = os.fsencode(opts.dest) - opts.sources = [os.fsencode(source) for source in opts.sources] + opts.distdir = os.fsencode(opts.distdir) + b'/' + opts.dest = os.fsencode(opts.dest) + opts.sources = [os.fsencode(source) for source in opts.sources] return opts diff --git a/bin/ebuild b/bin/ebuild index a57dd4941..d3961ddc5 100755 --- a/bin/ebuild +++ b/bin/ebuild @@ -144,9 +144,6 @@ if not os.path.isabs(ebuild): # the canonical path returned from os.getcwd() may may be unusable in # cases where the directory stucture is built from symlinks. pwd = os.environ.get('PWD', '') - if sys.hexversion < 0x3000000: - pwd = _unicode_decode(pwd, encoding=_encodings['content'], - errors='strict') if pwd and pwd != mycwd and \ os.path.realpath(pwd) == mycwd: mycwd = portage.normalize_path(pwd) @@ -163,16 +160,8 @@ vdb_path = os.path.realpath(os.path.join(portage.settings['EROOT'], VDB_PATH)) if ebuild_portdir != vdb_path and \ ebuild_portdir not in portage.portdb.porttrees: portdir_overlay = portage.settings.get("PORTDIR_OVERLAY", "") - if sys.hexversion >= 0x3000000: - os.environ["PORTDIR_OVERLAY"] = \ - portdir_overlay + \ - " " + _shell_quote(ebuild_portdir) - else: - os.environ["PORTDIR_OVERLAY"] = \ - _unicode_encode(portdir_overlay, - encoding=_encodings['content'], errors='strict') + \ - " " + _unicode_encode(_shell_quote(ebuild_portdir), - encoding=_encodings['content'], errors='strict') + os.environ["PORTDIR_OVERLAY"] = ( + portdir_overlay + " " + _shell_quote(ebuild_portdir)) print("Appending %s to PORTDIR_OVERLAY..." % ebuild_portdir) portage._reset_legacy_globals() diff --git a/bin/ebuild-ipc.py b/bin/ebuild-ipc.py index 773054f5d..c523572a7 100755 --- a/bin/ebuild-ipc.py +++ b/bin/ebuild-ipc.py @@ -100,7 +100,7 @@ class FifoWriter(AbstractPollTask): os.close(self._fd) self._fd = None -class EbuildIpc(object): +class EbuildIpc: # Timeout for each individual communication attempt (we retry # as long as the daemon process appears to be alive). diff --git a/bin/ecompress b/bin/ecompress index 2d74ed07a..7991bcfbe 100755 --- a/bin/ecompress +++ b/bin/ecompress @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2018 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH}"/isolated-functions.sh || exit 1 @@ -19,28 +19,30 @@ while [[ $# -gt 0 ]] ; do shift skip_dirs=() - skip_files=() + > "${T}/.ecompress_skip_files" || die for skip; do if [[ -d ${ED%/}/${skip#/} ]]; then skip_dirs+=( "${ED%/}/${skip#/}" ) else rm -f "${ED%/}/${skip#/}.ecompress" || die - skip_files+=("${ED%/}/${skip#/}") + printf -- '%s\n' "${EPREFIX}/${skip#/}" >> "${T}/.ecompress_skip_files" || die fi done if [[ ${#skip_dirs[@]} -gt 0 ]]; then - while read -r -d ''; do - skip_files+=(${REPLY#.ecompress}) + while read -r -d '' skip; do + skip=${skip%.ecompress} + printf -- '%s\n' "${skip#${D%/}}" >> "${T}/.ecompress_skip_files" || die done < <(find "${skip_dirs[@]}" -name '*.ecompress' -print0 -delete || die) fi - if [[ ${#skip_files[@]} -gt 0 && -s ${T}/.ecompress_had_precompressed ]]; then - sed_args=() - for f in "${skip_files[@]}"; do - sed_args+=(-e "s|^${f}\$||") - done - sed "${sed_args[@]}" -e '/^$/d' -i "${T}/.ecompress_had_precompressed" || die + if [[ -s ${T}/.ecompress_skip_files && -s ${T}/.ecompress_had_precompressed ]]; then + # Filter skipped files from ${T}/.ecompress_had_precompressed, + # using temporary files since these lists can be extremely large. + LC_COLLATE=C sort -u "${T}/.ecompress_skip_files" > "${T}/.ecompress_skip_files_sorted" || die + LC_COLLATE=C sort -u "${T}/.ecompress_had_precompressed" > "${T}/.ecompress_had_precompressed_sorted" || die + LC_COLLATE=C comm -13 "${T}/.ecompress_skip_files_sorted" "${T}/.ecompress_had_precompressed_sorted" > "${T}/.ecompress_had_precompressed" || die + rm -f "${T}/.ecompress_had_precompressed_sorted" "${T}/.ecompress_skip_files"{,_sorted} fi exit 0 @@ -80,7 +82,7 @@ while [[ $# -gt 0 ]] ; do continue 2 fi done - echo "${path}" >> "${T}"/.ecompress_had_precompressed + printf -- '%s\n' "${path#${D%/}}" >> "${T}"/.ecompress_had_precompressed || die ;; esac @@ -88,7 +90,7 @@ while [[ $# -gt 0 ]] ; do done < <(find "${find_args[@]}" -print0 || die) if [[ ${#collisions[@]} -gt 0 ]]; then - eqawarn "Colliding files found by ecompress:" + eqawarn "QA Notice: Colliding files found by ecompress:" eqawarn for x in "${!collisions[@]}"; do eqawarn " ${x}" @@ -189,13 +191,13 @@ find "${ED}" -name '*.ecompress' -delete -print0 | ret=${?} if [[ -s ${T}/.ecompress_had_precompressed ]]; then - eqawarn "One or more compressed files were found in docompress-ed directories." - eqawarn "Please fix the ebuild not to install compressed files (manpages," - eqawarn "documentation) when automatic compression is used:" + eqawarn "QA Notice: One or more compressed files were found in docompress-ed" + eqawarn "directories. Please fix the ebuild not to install compressed files" + eqawarn "(manpages, documentation) when automatic compression is used:" eqawarn n=0 while read -r f; do - eqawarn " ${f#${D%/}}" + eqawarn " ${f}" if [[ $(( n++ )) -eq 10 ]]; then eqawarn " ..." break diff --git a/bin/egencache b/bin/egencache index 04c8af2f0..1dc94b790 100755 --- a/bin/egencache +++ b/bin/egencache @@ -3,7 +3,7 @@ # Distributed under the terms of the GNU General Public License v2 # unicode_literals for compat with TextIOWrapper in Python 2 -from __future__ import print_function, unicode_literals +from __future__ import print_function import argparse import platform @@ -75,9 +75,6 @@ else: else: from portage.xml.metadata import parse_metadata_use -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - long = int def parse_args(args): usage = "egencache [options] <action> ... [atom] ..." @@ -241,7 +238,7 @@ def parse_args(args): return parser, options, args -class GenCache(object): +class GenCache: def __init__(self, portdb, cp_iter=None, max_jobs=None, max_load=None, rsync=False): # The caller must set portdb.porttrees in order to constrain @@ -344,7 +341,7 @@ class GenCache(object): max_mtime = ec_hash.mtime if max_mtime == sc.mtime: max_mtime += 1 - max_mtime = long(max_mtime) + max_mtime = int(max_mtime) try: os.utime(ebuild_hash.location, (max_mtime, max_mtime)) except OSError as e: @@ -450,7 +447,7 @@ class GenCache(object): if hasattr(trg_cache, '_prune_empty_dirs'): trg_cache._prune_empty_dirs() -class GenPkgDescIndex(object): +class GenPkgDescIndex: def __init__(self, portdb, output_file): self.returncode = os.EX_OK self._portdb = portdb @@ -473,7 +470,7 @@ class GenPkgDescIndex(object): f.close() -class GenUseLocalDesc(object): +class GenUseLocalDesc: def __init__(self, portdb, output=None, preserve_comments=False): self.returncode = os.EX_OK @@ -668,7 +665,7 @@ class GenUseLocalDesc(object): os.utime(desc_path, (mtime, mtime)) -class GenChangeLogs(object): +class GenChangeLogs: def __init__(self, portdb, changelog_output, changelog_reversed, max_jobs=None, max_load=None): self.returncode = os.EX_OK @@ -709,7 +706,7 @@ class GenChangeLogs(object): # This cp has not been added to the repo. return - lmod = long(lmod) + lmod = int(lmod) try: cmod = os.stat('ChangeLog')[stat.ST_MTIME] diff --git a/bin/filter-bash-environment.py b/bin/filter-bash-environment.py index 06cac7214..7820538ef 100755 --- a/bin/filter-bash-environment.py +++ b/bin/filter-bash-environment.py @@ -136,14 +136,9 @@ if __name__ == "__main__": sys.stderr.flush() sys.exit(2) - file_in = sys.stdin - file_out = sys.stdout - if sys.hexversion >= 0x3000000: - file_in = sys.stdin.buffer - file_out = sys.stdout.buffer - var_pattern = os.fsencode(args[0]).split() - else: - var_pattern = args[0].split() + file_in = sys.stdin.buffer + file_out = sys.stdout.buffer + var_pattern = os.fsencode(args[0]).split() # Filter invalid variable names that are not supported by bash. var_pattern.append(br'\d.*') diff --git a/bin/glsa-check b/bin/glsa-check index f9dae110f..64a4ea617 100755 --- a/bin/glsa-check +++ b/bin/glsa-check @@ -2,7 +2,7 @@ # Copyright 1999-2019 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function, unicode_literals +from __future__ import print_function import argparse import re diff --git a/bin/install.py b/bin/install.py index 03a4511bb..2dc0ccc4c 100755 --- a/bin/install.py +++ b/bin/install.py @@ -232,16 +232,15 @@ def main(args): cmdline = [install_binary] cmdline += args - if sys.hexversion >= 0x3000000: - # We can't trust that the filesystem encoding (locale dependent) - # correctly matches the arguments, so use surrogateescape to - # pass through the original argv bytes for Python 3. - fs_encoding = sys.getfilesystemencoding() - cmdline = [x.encode(fs_encoding, 'surrogateescape') for x in cmdline] - files = [x.encode(fs_encoding, 'surrogateescape') for x in files] - if opts.target_directory is not None: - opts.target_directory = \ - opts.target_directory.encode(fs_encoding, 'surrogateescape') + # We can't trust that the filesystem encoding (locale dependent) + # correctly matches the arguments, so use surrogateescape to + # pass through the original argv bytes for Python 3. + fs_encoding = sys.getfilesystemencoding() + cmdline = [x.encode(fs_encoding, 'surrogateescape') for x in cmdline] + files = [x.encode(fs_encoding, 'surrogateescape') for x in files] + if opts.target_directory is not None: + opts.target_directory = \ + opts.target_directory.encode(fs_encoding, 'surrogateescape') returncode = subprocess.call(cmdline) if returncode == os.EX_OK: diff --git a/bin/pid-ns-init b/bin/pid-ns-init index 18c74f799..3a218a5df 100644 --- a/bin/pid-ns-init +++ b/bin/pid-ns-init @@ -39,7 +39,7 @@ def preexec_fn(uid, gid, groups, umask): os.umask(umask) # CPython >= 3 subprocess.Popen handles this internally. - if sys.version_info.major < 3 or platform.python_implementation() != 'CPython': + if platform.python_implementation() != 'CPython': for signum in ( signal.SIGHUP, signal.SIGINT, @@ -70,10 +70,10 @@ def main(argv): groups = tuple(int(group) for group in groups.split(',')) if groups else None umask = int(umask) if umask else None - popen_kwargs = {} - popen_kwargs['preexec_fn'] = functools.partial(preexec_fn, uid, gid, groups, umask) - if sys.version_info.major > 2: - popen_kwargs['pass_fds'] = pass_fds + popen_kwargs = { + 'preexec_fn': functools.partial(preexec_fn, uid, gid, groups, umask), + 'pass_fds': pass_fds, + } # Isolate parent process from process group SIGSTOP (bug 675870) setsid = True os.setsid() diff --git a/bin/portageq b/bin/portageq index 2d572b70c..d54f8d02b 100755 --- a/bin/portageq +++ b/bin/portageq @@ -2,7 +2,7 @@ # Copyright 1999-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function, unicode_literals +from __future__ import print_function import argparse import signal @@ -1020,7 +1020,7 @@ docstrings['list_preserved_libs'] = """<eroot> list_preserved_libs.__doc__ = docstrings['list_preserved_libs'] -class MaintainerEmailMatcher(object): +class MaintainerEmailMatcher: def __init__(self, maintainer_emails): self._re = re.compile("^(%s)$" % "|".join(maintainer_emails), re.I) diff --git a/bin/quickpkg b/bin/quickpkg index b9ce52f8a..17de837f7 100755 --- a/bin/quickpkg +++ b/bin/quickpkg @@ -137,7 +137,7 @@ def quickpkg_atom(options, infos, arg, eout): missing_package = compression["package"] eout.eerror("File compression unsupported %s. Missing package: %s" % (binpkg_compression, missing_package)) return 1 - cmd = [varexpand(x, mydict=settings) for x in shlex_split(compression["compress"])] + cmd = shlex_split(varexpand(compression["compress"], mydict=settings)) # Filter empty elements that make Popen fail cmd = [x for x in cmd if x != ""] with open(binpkg_tmpfile, "wb") as fobj: diff --git a/bin/socks5-server.py b/bin/socks5-server.py index 1d07c98ed..8a1a4d1be 100644 --- a/bin/socks5-server.py +++ b/bin/socks5-server.py @@ -24,7 +24,7 @@ except AttributeError: current_task = asyncio.Task.current_task -class Socks5Server(object): +class Socks5Server: """ An asynchronous SOCKSv5 server. """ diff --git a/bin/xattr-helper.py b/bin/xattr-helper.py index a8aef3880..7ece98027 100755 --- a/bin/xattr-helper.py +++ b/bin/xattr-helper.py @@ -26,24 +26,14 @@ _UNQUOTE_RE = re.compile(br'\\[0-7]{3}') _FS_ENCODING = sys.getfilesystemencoding() -if sys.hexversion < 0x3000000: +def octal_quote_byte(b): + return ('\\%03o' % ord(b)).encode('ascii') - def octal_quote_byte(b): - return b'\\%03o' % ord(b) - def unicode_encode(s): - if isinstance(s, unicode): - s = s.encode(_FS_ENCODING) - return s -else: - - def octal_quote_byte(b): - return ('\\%03o' % ord(b)).encode('ascii') - - def unicode_encode(s): - if isinstance(s, str): - s = s.encode(_FS_ENCODING, 'surrogateescape') - return s +def unicode_encode(s): + if isinstance(s, str): + s = s.encode(_FS_ENCODING, 'surrogateescape') + return s def quote(s, quote_chars): @@ -157,20 +147,14 @@ def main(argv): options = parser.parse_args(argv) - if sys.hexversion >= 0x3000000: - file_in = sys.stdin.buffer.raw - else: - file_in = sys.stdin + file_in = sys.stdin.buffer.raw if options.dump: if options.paths: options.paths = [unicode_encode(x) for x in options.paths] else: options.paths = [x for x in file_in.read().split(b'\0') if x] - if sys.hexversion >= 0x3000000: - file_out = sys.stdout.buffer - else: - file_out = sys.stdout + file_out = sys.stdout.buffer dump_xattrs(options.paths, file_out) elif options.restore: diff --git a/lib/_emerge/AbstractEbuildProcess.py b/lib/_emerge/AbstractEbuildProcess.py index 1c1955cfe..ae1aae55f 100644 --- a/lib/_emerge/AbstractEbuildProcess.py +++ b/lib/_emerge/AbstractEbuildProcess.py @@ -1,4 +1,4 @@ -# Copyright 1999-2019 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import errno @@ -196,6 +196,7 @@ class AbstractEbuildProcess(SpawnProcess): null_fd = os.open('/dev/null', os.O_RDONLY) self.fd_pipes[0] = null_fd + self.log_filter_file = self.settings.get('PORTAGE_LOG_FILTER_FILE_CMD') try: SpawnProcess._start(self) finally: diff --git a/lib/_emerge/AbstractPollTask.py b/lib/_emerge/AbstractPollTask.py index 7e9f2b536..661b81616 100644 --- a/lib/_emerge/AbstractPollTask.py +++ b/lib/_emerge/AbstractPollTask.py @@ -3,7 +3,6 @@ import array import errno -import logging import os from portage.util import writemsg_level diff --git a/lib/_emerge/AsynchronousLock.py b/lib/_emerge/AsynchronousLock.py index aed1bcb15..d2a6773ff 100644 --- a/lib/_emerge/AsynchronousLock.py +++ b/lib/_emerge/AsynchronousLock.py @@ -1,4 +1,4 @@ -# Copyright 2010-2018 Gentoo Foundation +# Copyright 2010-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import fcntl @@ -192,16 +192,6 @@ class _LockProcess(AbstractPollTask): fcntl.fcntl(in_pr, fcntl.F_SETFL, fcntl.fcntl(in_pr, fcntl.F_GETFL) | os.O_NONBLOCK) - # FD_CLOEXEC is enabled by default in Python >=3.4. - if sys.hexversion < 0x3040000: - try: - fcntl.FD_CLOEXEC - except AttributeError: - pass - else: - fcntl.fcntl(in_pr, fcntl.F_SETFD, - fcntl.fcntl(in_pr, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) - self.scheduler.add_reader(in_pr, self._output_handler) self._registered = True self._proc = SpawnProcess( diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py index 36d027de3..218d4d2ab 100644 --- a/lib/_emerge/BinpkgFetcher.py +++ b/lib/_emerge/BinpkgFetcher.py @@ -1,4 +1,4 @@ -# Copyright 1999-2018 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import functools @@ -6,10 +6,7 @@ import functools from _emerge.AsynchronousLock import AsynchronousLock from _emerge.CompositeTask import CompositeTask from _emerge.SpawnProcess import SpawnProcess -try: - from urllib.parse import urlparse as urllib_parse_urlparse -except ImportError: - from urlparse import urlparse as urllib_parse_urlparse +from urllib.parse import urlparse as urllib_parse_urlparse import stat import sys import portage @@ -17,9 +14,6 @@ from portage import os from portage.util._async.AsyncTaskFuture import AsyncTaskFuture from portage.util._pty import _create_pty_or_pipe -if sys.hexversion >= 0x3000000: - long = int - class BinpkgFetcher(CompositeTask): @@ -158,6 +152,7 @@ class _BinpkgFetcherProcess(SpawnProcess): self.env = fetch_env if settings.selinux_enabled(): self._selinux_type = settings["PORTAGE_FETCH_T"] + self.log_filter_file = settings.get('PORTAGE_LOG_FILTER_FILE_CMD') SpawnProcess._start(self) def _pipe(self, fd_pipes): @@ -184,7 +179,7 @@ class _BinpkgFetcherProcess(SpawnProcess): self.pkg.cpv)].get("_mtime_") if remote_mtime is not None: try: - remote_mtime = long(remote_mtime) + remote_mtime = int(remote_mtime) except ValueError: pass else: @@ -237,4 +232,3 @@ class _BinpkgFetcherProcess(SpawnProcess): self._lock_obj = None self.locked = False return result - diff --git a/lib/_emerge/BinpkgPrefetcher.py b/lib/_emerge/BinpkgPrefetcher.py index 7ca897049..1393297b5 100644 --- a/lib/_emerge/BinpkgPrefetcher.py +++ b/lib/_emerge/BinpkgPrefetcher.py @@ -40,4 +40,3 @@ class BinpkgPrefetcher(CompositeTask): self._current_task = None self.returncode = os.EX_OK self.wait() - diff --git a/lib/_emerge/BlockerCache.py b/lib/_emerge/BlockerCache.py index 53342d6d6..8154d9ade 100644 --- a/lib/_emerge/BlockerCache.py +++ b/lib/_emerge/BlockerCache.py @@ -1,24 +1,12 @@ -# Copyright 1999-2013 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import errno -import sys from portage.util import writemsg from portage.data import secpass import portage from portage import os - -try: - import cPickle as pickle -except ImportError: - import pickle - -if sys.hexversion >= 0x3000000: - basestring = str - long = int - _unicode = str -else: - _unicode = unicode +import pickle class BlockerCache(portage.cache.mappings.MutableMapping): """This caches blockers of installed packages so that dep_check does not @@ -32,7 +20,7 @@ class BlockerCache(portage.cache.mappings.MutableMapping): # it's wasteful to update it for every vdb change. _cache_threshold = 5 - class BlockerData(object): + class BlockerData: __slots__ = ("__weakref__", "atoms", "counter") @@ -82,7 +70,7 @@ class BlockerCache(portage.cache.mappings.MutableMapping): # corruption is detected as soon as possible. invalid_items = set() for k, v in self._cache_data["blockers"].items(): - if not isinstance(k, basestring): + if not isinstance(k, str): invalid_items.add(k) continue try: @@ -97,7 +85,7 @@ class BlockerCache(portage.cache.mappings.MutableMapping): invalid_items.add(k) continue counter, atoms = v - if not isinstance(counter, (int, long)): + if not isinstance(counter, int): invalid_items.add(k) continue if not isinstance(atoms, (list, tuple)): @@ -105,7 +93,7 @@ class BlockerCache(portage.cache.mappings.MutableMapping): continue invalid_atom = False for atom in atoms: - if not isinstance(atom, basestring): + if not isinstance(atom, str): invalid_atom = True break if atom[:1] != "!" or \ @@ -164,8 +152,8 @@ class BlockerCache(portage.cache.mappings.MutableMapping): @param blocker_data: An object with counter and atoms attributes. @type blocker_data: BlockerData """ - self._cache_data["blockers"][_unicode(cpv)] = (blocker_data.counter, - tuple(_unicode(x) for x in blocker_data.atoms)) + self._cache_data["blockers"][str(cpv)] = (blocker_data.counter, + tuple(str(x) for x in blocker_data.atoms)) self._modified.add(cpv) def __iter__(self): @@ -188,4 +176,3 @@ class BlockerCache(portage.cache.mappings.MutableMapping): @return: An object with counter and atoms attributes. """ return self.BlockerData(*self._cache_data["blockers"][cpv]) - diff --git a/lib/_emerge/BlockerDB.py b/lib/_emerge/BlockerDB.py index 5b3b01c37..7e1f88615 100644 --- a/lib/_emerge/BlockerDB.py +++ b/lib/_emerge/BlockerDB.py @@ -1,7 +1,6 @@ -# Copyright 1999-2018 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -import sys import portage from portage import os @@ -13,10 +12,8 @@ from _emerge.BlockerCache import BlockerCache from _emerge.Package import Package from _emerge.show_invalid_depstring_notice import show_invalid_depstring_notice -if sys.hexversion >= 0x3000000: - long = int -class BlockerDB(object): +class BlockerDB: def __init__(self, fake_vartree): root_config = fake_vartree._root_config diff --git a/lib/_emerge/DepPriority.py b/lib/_emerge/DepPriority.py index 34fdb481c..ec79bb3d5 100644 --- a/lib/_emerge/DepPriority.py +++ b/lib/_emerge/DepPriority.py @@ -53,4 +53,3 @@ class DepPriority(AbstractDepPriority): if self.runtime_post: return "runtime_post" return "soft" - diff --git a/lib/_emerge/DepPriorityNormalRange.py b/lib/_emerge/DepPriorityNormalRange.py index 86395549f..5f3f3da70 100644 --- a/lib/_emerge/DepPriorityNormalRange.py +++ b/lib/_emerge/DepPriorityNormalRange.py @@ -2,7 +2,7 @@ # Distributed under the terms of the GNU General Public License v2 from _emerge.DepPriority import DepPriority -class DepPriorityNormalRange(object): +class DepPriorityNormalRange: """ DepPriority properties Index Category diff --git a/lib/_emerge/DepPrioritySatisfiedRange.py b/lib/_emerge/DepPrioritySatisfiedRange.py index 391f5409b..e056e676f 100644 --- a/lib/_emerge/DepPrioritySatisfiedRange.py +++ b/lib/_emerge/DepPrioritySatisfiedRange.py @@ -2,7 +2,7 @@ # Distributed under the terms of the GNU General Public License v2 from _emerge.DepPriority import DepPriority -class DepPrioritySatisfiedRange(object): +class DepPrioritySatisfiedRange: """ DepPriority Index Category diff --git a/lib/_emerge/Dependency.py b/lib/_emerge/Dependency.py index 2ec860f83..ab3c5bbff 100644 --- a/lib/_emerge/Dependency.py +++ b/lib/_emerge/Dependency.py @@ -18,4 +18,3 @@ class Dependency(SlotObject): self.collapsed_parent = self.parent if self.collapsed_priority is None: self.collapsed_priority = self.priority - diff --git a/lib/_emerge/DependencyArg.py b/lib/_emerge/DependencyArg.py index 29a0072c4..f7ac879ec 100644 --- a/lib/_emerge/DependencyArg.py +++ b/lib/_emerge/DependencyArg.py @@ -1,13 +1,10 @@ -# Copyright 1999-2013 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - -import sys from portage import _encodings, _unicode_encode -class DependencyArg(object): +class DependencyArg: __slots__ = ('arg', 'force_reinstall', 'internal', 'reset_depth', 'root_config') @@ -33,14 +30,4 @@ class DependencyArg(object): return hash((self.arg, self.root_config.root)) def __str__(self): - # Use unicode_literals format string for python-2.x safety, - # ensuring that self.arg.__unicode__() is used - # when necessary. return "%s" % (self.arg,) - - if sys.hexversion < 0x3000000: - - __unicode__ = __str__ - - def __str__(self): - return _unicode_encode(self.__unicode__(), encoding=_encodings['content']) diff --git a/lib/_emerge/EbuildBuild.py b/lib/_emerge/EbuildBuild.py index ab5a4da74..4da815988 100644 --- a/lib/_emerge/EbuildBuild.py +++ b/lib/_emerge/EbuildBuild.py @@ -1,8 +1,6 @@ # Copyright 1999-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - import functools import io @@ -161,18 +159,18 @@ class EbuildBuild(CompositeTask): settings=self.settings), self._default_final_exit) return - else: - fetcher = EbuildFetcher( - config_pool=self.config_pool, - ebuild_path=self._ebuild_path, - fetchall=self.opts.fetch_all_uri, - fetchonly=self.opts.fetchonly, - background=False, - logfile=None, - pkg=self.pkg, - scheduler=self.scheduler) - self._start_task(fetcher, self._fetchonly_exit) - return + + fetcher = EbuildFetcher( + config_pool=self.config_pool, + ebuild_path=self._ebuild_path, + fetchall=self.opts.fetch_all_uri, + fetchonly=self.opts.fetchonly, + background=False, + logfile=None, + pkg=self.pkg, + scheduler=self.scheduler) + self._start_task(fetcher, self._fetchonly_exit) + return self._build_dir = EbuildBuildDir( scheduler=self.scheduler, settings=settings) diff --git a/lib/_emerge/EbuildBuildDir.py b/lib/_emerge/EbuildBuildDir.py index 477113db8..04667dbd7 100644 --- a/lib/_emerge/EbuildBuildDir.py +++ b/lib/_emerge/EbuildBuildDir.py @@ -158,4 +158,3 @@ class EbuildBuildDir(SlotObject): class AlreadyLocked(portage.exception.PortageException): pass - diff --git a/lib/_emerge/EbuildExecuter.py b/lib/_emerge/EbuildExecuter.py index acc57c1d0..11e49804f 100644 --- a/lib/_emerge/EbuildExecuter.py +++ b/lib/_emerge/EbuildExecuter.py @@ -79,4 +79,3 @@ class EbuildExecuter(CompositeTask): settings=self.settings)) self._start_task(ebuild_phases, self._default_final_exit) - diff --git a/lib/_emerge/EbuildFetcher.py b/lib/_emerge/EbuildFetcher.py index 1e40994fb..107a6d590 100644 --- a/lib/_emerge/EbuildFetcher.py +++ b/lib/_emerge/EbuildFetcher.py @@ -1,4 +1,4 @@ -# Copyright 1999-2018 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import copy @@ -225,6 +225,7 @@ class _EbuildFetcherProcess(ForkProcess): settings["NOCOLOR"] = nocolor self._settings = settings + self.log_filter_file = settings.get('PORTAGE_LOG_FILTER_FILE_CMD') ForkProcess._start(self) # Free settings now since it's no longer needed in @@ -363,16 +364,11 @@ class _EbuildFetcherProcess(ForkProcess): if msg: self.scheduler.output(msg, log_path=self.logfile) - def _async_waitpid_cb(self, *args, **kwargs): + def _proc_join_done(self, proc, future): """ - Override _async_waitpid_cb to perform cleanup that is - not necessarily idempotent. + Extend _proc_join_done to emit an eerror message for fetch failure. """ - ForkProcess._async_waitpid_cb(self, *args, **kwargs) - # Collect elog messages that might have been - # created by the pkg_nofetch phase. - # Skip elog messages for prefetch, in order to avoid duplicates. - if not self.prefetch and self.returncode != os.EX_OK: + if not self.prefetch and not future.cancelled() and proc.exitcode != os.EX_OK: msg_lines = [] msg = "Fetch failed for '%s'" % (self.pkg.cpv,) if self.logfile is not None: @@ -381,3 +377,4 @@ class _EbuildFetcherProcess(ForkProcess): if self.logfile is not None: msg_lines.append(" '%s'" % (self.logfile,)) self._eerror(msg_lines) + super(_EbuildFetcherProcess, self)._proc_join_done(proc, future) diff --git a/lib/_emerge/EbuildMetadataPhase.py b/lib/_emerge/EbuildMetadataPhase.py index efe71892c..d00f194c2 100644 --- a/lib/_emerge/EbuildMetadataPhase.py +++ b/lib/_emerge/EbuildMetadataPhase.py @@ -1,4 +1,4 @@ -# Copyright 1999-2018 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from _emerge.SubProcess import SubProcess @@ -93,16 +93,6 @@ class EbuildMetadataPhase(SubProcess): fcntl.fcntl(master_fd, fcntl.F_SETFL, fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK) - # FD_CLOEXEC is enabled by default in Python >=3.4. - if sys.hexversion < 0x3040000: - try: - fcntl.FD_CLOEXEC - except AttributeError: - pass - else: - fcntl.fcntl(master_fd, fcntl.F_SETFD, - fcntl.fcntl(master_fd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) - fd_pipes[slave_fd] = slave_fd settings["PORTAGE_PIPE_FD"] = str(slave_fd) diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py index 477e0ba97..e6256d0aa 100644 --- a/lib/_emerge/EbuildPhase.py +++ b/lib/_emerge/EbuildPhase.py @@ -1,8 +1,6 @@ # Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - import functools import gzip import io @@ -26,6 +24,8 @@ from portage.package.ebuild.prepare_build_dirs import (_prepare_workdir, from portage.util.futures.compat_coroutine import coroutine from portage.util import writemsg from portage.util._async.AsyncTaskFuture import AsyncTaskFuture +from portage.util._async.BuildLogger import BuildLogger +from portage.util.futures import asyncio from portage.util.futures.executor.fork import ForkExecutor try: @@ -45,7 +45,8 @@ portage.proxy.lazyimport.lazyimport(globals(), '_post_src_install_soname_symlinks,' + \ '_post_src_install_uid_fix,_postinst_bsdflags,' + \ '_post_src_install_write_metadata,' + \ - '_preinst_bsdflags' + '_preinst_bsdflags', + 'portage.util.futures.unix_events:_set_nonblocking', ) from portage import os from portage import _encodings @@ -69,6 +70,11 @@ class EbuildPhase(CompositeTask): _locked_phases = ("setup", "preinst", "postinst", "prerm", "postrm") def _start(self): + future = asyncio.ensure_future(self._async_start(), loop=self.scheduler) + self._start_task(AsyncTaskFuture(future=future), self._async_start_exit) + + @coroutine + def _async_start(self): need_builddir = self.phase not in EbuildProcess._phases_without_builddir @@ -126,7 +132,7 @@ class EbuildPhase(CompositeTask): # Force background=True for this header since it's intended # for the log and it doesn't necessarily need to be visible # elsewhere. - self._elog('einfo', msg, background=True) + yield self._elog('einfo', msg, background=True) if self.phase == 'package': if 'PORTAGE_BINPKG_TMPFILE' not in self.settings: @@ -134,6 +140,12 @@ class EbuildPhase(CompositeTask): os.path.join(self.settings['PKGDIR'], self.settings['CATEGORY'], self.settings['PF']) + '.tbz2' + def _async_start_exit(self, task): + task.future.cancelled() or task.future.result() + if self._default_exit(task) != os.EX_OK: + self.wait() + return + if self.phase in ("pretend", "prerm"): env_extractor = BinpkgEnvExtractor(background=self.background, scheduler=self.scheduler, settings=self.settings) @@ -391,6 +403,7 @@ class EbuildPhase(CompositeTask): self.returncode = 1 self.wait() + @coroutine def _elog(self, elog_funcname, lines, background=None): if background is None: background = self.background @@ -407,11 +420,31 @@ class EbuildPhase(CompositeTask): portage.output.havecolor = global_havecolor msg = out.getvalue() if msg: - log_path = None - if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": - log_path = self.settings.get("PORTAGE_LOG_FILE") - self.scheduler.output(msg, log_path=log_path, - background=background) + build_logger = None + try: + log_file = None + log_path = None + if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": + log_path = self.settings.get("PORTAGE_LOG_FILE") + if log_path: + build_logger = BuildLogger(env=self.settings.environ(), + log_path=log_path, + log_filter_file=self.settings.get('PORTAGE_LOG_FILTER_FILE_CMD'), + scheduler=self.scheduler) + build_logger.start() + _set_nonblocking(build_logger.stdin.fileno()) + log_file = build_logger.stdin + + yield self.scheduler.async_output(msg, log_file=log_file, + background=background) + + if build_logger is not None: + build_logger.stdin.close() + yield build_logger.async_wait() + except asyncio.CancelledError: + if build_logger is not None: + build_logger.cancel() + raise class _PostPhaseCommands(CompositeTask): @@ -480,4 +513,4 @@ class _PostPhaseCommands(CompositeTask): qa_msg.extend("\t%s: %s" % (filename, " ".join(sorted(soname_deps))) for filename, soname_deps in unresolved) qa_msg.append("") - self.elog("eqawarn", qa_msg) + yield self.elog("eqawarn", qa_msg) diff --git a/lib/_emerge/FakeVartree.py b/lib/_emerge/FakeVartree.py index 3f82e97e9..37349bcdd 100644 --- a/lib/_emerge/FakeVartree.py +++ b/lib/_emerge/FakeVartree.py @@ -1,9 +1,6 @@ -# Copyright 1999-2013 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - -import sys import warnings import portage @@ -19,13 +16,8 @@ from portage.update import grab_updates, parse_updates, update_dbentries from portage.versions import _pkg_str from _emerge.resolver.DbapiProvidesIndex import PackageDbapiProvidesIndex -if sys.hexversion >= 0x3000000: - long = int - _unicode = str -else: - _unicode = unicode -class FakeVardbGetPath(object): +class FakeVardbGetPath: """ Implements the vardbapi.getpath() method which is used in error handling code for the Package class and vartree.get_provide(). @@ -167,7 +159,7 @@ class FakeVartree(vartree): raise _DynamicDepsNotApplicable() for k, v in built_slot_operator_atoms.items(): live_metadata[k] += (" " + - " ".join(_unicode(atom) for atom in v)) + " ".join(str(atom) for atom in v)) self.dbapi.aux_update(pkg.cpv, live_metadata) except _DynamicDepsNotApplicable: @@ -249,7 +241,7 @@ class FakeVartree(vartree): if pkg is not None: counter, mtime = real_vardb.aux_get(cpv, validation_keys) try: - counter = long(counter) + counter = int(counter) except ValueError: counter = 0 diff --git a/lib/_emerge/FifoIpcDaemon.py b/lib/_emerge/FifoIpcDaemon.py index 2ec69d1cb..a21e94db6 100644 --- a/lib/_emerge/FifoIpcDaemon.py +++ b/lib/_emerge/FifoIpcDaemon.py @@ -1,13 +1,6 @@ -# Copyright 2010-2018 Gentoo Foundation +# Copyright 2010-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -import sys - -try: - import fcntl -except ImportError: - # http://bugs.jython.org/issue1074 - fcntl = None from portage import os from _emerge.AbstractPollTask import AbstractPollTask @@ -28,17 +21,6 @@ class FifoIpcDaemon(AbstractPollTask): self._files.pipe_in = \ os.open(self.input_fifo, os.O_RDONLY|os.O_NONBLOCK) - # FD_CLOEXEC is enabled by default in Python >=3.4. - if sys.hexversion < 0x3040000 and fcntl is not None: - try: - fcntl.FD_CLOEXEC - except AttributeError: - pass - else: - fcntl.fcntl(self._files.pipe_in, fcntl.F_SETFD, - fcntl.fcntl(self._files.pipe_in, - fcntl.F_GETFD) | fcntl.FD_CLOEXEC) - self.scheduler.add_reader( self._files.pipe_in, self._input_handler) @@ -55,17 +37,6 @@ class FifoIpcDaemon(AbstractPollTask): self._files.pipe_in = \ os.open(self.input_fifo, os.O_RDONLY|os.O_NONBLOCK) - # FD_CLOEXEC is enabled by default in Python >=3.4. - if sys.hexversion < 0x3040000 and fcntl is not None: - try: - fcntl.FD_CLOEXEC - except AttributeError: - pass - else: - fcntl.fcntl(self._files.pipe_in, fcntl.F_SETFD, - fcntl.fcntl(self._files.pipe_in, - fcntl.F_GETFD) | fcntl.FD_CLOEXEC) - self.scheduler.add_reader( self._files.pipe_in, self._input_handler) diff --git a/lib/_emerge/JobStatusDisplay.py b/lib/_emerge/JobStatusDisplay.py index b8e142af9..8986e90db 100644 --- a/lib/_emerge/JobStatusDisplay.py +++ b/lib/_emerge/JobStatusDisplay.py @@ -1,8 +1,6 @@ -# Copyright 1999-2015 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - import io import sys import time @@ -16,10 +14,7 @@ from portage.output import xtermTitle from _emerge.getloadavg import getloadavg -if sys.hexversion >= 0x3000000: - basestring = str - -class JobStatusDisplay(object): +class JobStatusDisplay: _bound_properties = ("curval", "failed", "running") @@ -61,7 +56,7 @@ class JobStatusDisplay(object): object.__setattr__(self, "_term_codes", term_codes) encoding = sys.getdefaultencoding() for k, v in self._term_codes.items(): - if not isinstance(v, basestring): + if not isinstance(v, str): self._term_codes[k] = v.decode(encoding, 'replace') if self._isatty: @@ -88,9 +83,7 @@ class JobStatusDisplay(object): # avoid potential UnicodeEncodeError s = _unicode_encode(s, encoding=_encodings['stdio'], errors='backslashreplace') - out = self.out - if sys.hexversion >= 0x3000000: - out = out.buffer + out = self.out.buffer out.write(s) out.flush() diff --git a/lib/_emerge/MergeListItem.py b/lib/_emerge/MergeListItem.py index 938f8014a..3b65b16a2 100644 --- a/lib/_emerge/MergeListItem.py +++ b/lib/_emerge/MergeListItem.py @@ -85,7 +85,7 @@ class MergeListItem(CompositeTask): self._start_task(build, self._default_final_exit) return - elif pkg.type_name == "binary": + if pkg.type_name == "binary": binpkg = Binpkg(background=self.background, find_blockers=find_blockers, diff --git a/lib/_emerge/Package.py b/lib/_emerge/Package.py index b2dfc07c1..1a837cb44 100644 --- a/lib/_emerge/Package.py +++ b/lib/_emerge/Package.py @@ -1,10 +1,6 @@ -# Copyright 1999-2018 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - -import functools -import sys from itertools import chain import warnings @@ -21,13 +17,6 @@ from portage.exception import InvalidData, InvalidDependString from portage.localization import _ from _emerge.Task import Task -if sys.hexversion >= 0x3000000: - basestring = str - long = int - _unicode = str -else: - _unicode = unicode - class Package(Task): __hash__ = Task.__hash__ @@ -222,7 +211,7 @@ class Package(Task): else: raise TypeError("root_config argument is required") - elements = [type_name, root, _unicode(cpv), operation] + elements = [type_name, root, str(cpv), operation] # For installed (and binary) packages we don't care for the repo # when it comes to hashing, because there can only be one cpv. @@ -515,7 +504,7 @@ class Package(Task): cpv_color = "PKG_NOMERGE" build_id_str = "" - if isinstance(self.cpv.build_id, long) and self.cpv.build_id > 0: + if isinstance(self.cpv.build_id, int) and self.cpv.build_id > 0: build_id_str = "-%s" % self.cpv.build_id s = "(%s, %s" \ @@ -537,15 +526,7 @@ class Package(Task): s += ")" return s - if sys.hexversion < 0x3000000: - - __unicode__ = __str__ - - def __str__(self): - return _unicode_encode(self.__unicode__(), - encoding=_encodings['content']) - - class _use_class(object): + class _use_class: __slots__ = ("enabled", "_expand", "_expand_hidden", "_force", "_pkg", "_mask") @@ -670,7 +651,7 @@ class Package(Task): return use_str - class _iuse(object): + class _iuse: __slots__ = ("__weakref__", "_iuse_implicit_match", "_pkg", "alias_mapping", "all", "all_aliases", "enabled", "disabled", "tokens") @@ -714,7 +695,7 @@ class Package(Task): @return: True if all flags are valid USE values which may be specified in USE dependencies, False otherwise. """ - if isinstance(flags, basestring): + if isinstance(flags, str): flags = [flags] for flag in flags: @@ -727,7 +708,7 @@ class Package(Task): """ @return: A list of flags missing from IUSE. """ - if isinstance(flags, basestring): + if isinstance(flags, str): flags = [flags] missing_iuse = [] for flag in flags: @@ -743,7 +724,8 @@ class Package(Task): """ if flag in self.all: return flag - elif flag in self.all_aliases: + + if flag in self.all_aliases: for k, v in self.alias_mapping.items(): if flag in v: return k @@ -873,14 +855,14 @@ class _PackageMetadataWrapper(_PackageMetadataWrapperBase): getattr(self, "_set_" + k.lower())(k, v) def _set_inherited(self, k, v): - if isinstance(v, basestring): + if isinstance(v, str): v = frozenset(v.split()) self._pkg.inherited = v def _set_counter(self, k, v): - if isinstance(v, basestring): + if isinstance(v, str): try: - v = long(v.strip()) + v = int(v.strip()) except ValueError: v = 0 self._pkg.counter = v @@ -898,9 +880,9 @@ class _PackageMetadataWrapper(_PackageMetadataWrapperBase): pass def _set__mtime_(self, k, v): - if isinstance(v, basestring): + if isinstance(v, str): try: - v = long(v.strip()) + v = int(v.strip()) except ValueError: v = 0 self._pkg.mtime = v diff --git a/lib/_emerge/PackageVirtualDbapi.py b/lib/_emerge/PackageVirtualDbapi.py index 26293dd98..4bba8c959 100644 --- a/lib/_emerge/PackageVirtualDbapi.py +++ b/lib/_emerge/PackageVirtualDbapi.py @@ -1,7 +1,6 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -import sys from portage.dbapi import dbapi from portage.dbapi.dep_expand import dep_expand @@ -41,9 +40,6 @@ class PackageVirtualDbapi(dbapi): def __bool__(self): return bool(self._cpv_map) - if sys.hexversion < 0x3000000: - __nonzero__ = __bool__ - def __iter__(self): return iter(self._cpv_map.values()) @@ -146,4 +142,3 @@ class PackageVirtualDbapi(dbapi): def aux_update(self, cpv, values): self._cpv_map[cpv]._metadata.update(values) self._clear_cache() - diff --git a/lib/_emerge/PipeReader.py b/lib/_emerge/PipeReader.py index 1aa5ee3bf..852378113 100644 --- a/lib/_emerge/PipeReader.py +++ b/lib/_emerge/PipeReader.py @@ -1,8 +1,7 @@ -# Copyright 1999-2018 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import fcntl -import sys from portage import os from _emerge.AbstractPollTask import AbstractPollTask @@ -27,16 +26,6 @@ class PipeReader(AbstractPollTask): fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK) - # FD_CLOEXEC is enabled by default in Python >=3.4. - if sys.hexversion < 0x3040000: - try: - fcntl.FD_CLOEXEC - except AttributeError: - pass - else: - fcntl.fcntl(fd, fcntl.F_SETFD, - fcntl.fcntl(fd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) - if self._use_array: self.scheduler.add_reader(fd, self._array_output_handler, f) else: @@ -103,4 +92,3 @@ class PipeReader(AbstractPollTask): self.scheduler.remove_reader(f.fileno()) f.close() self.input_files = None - diff --git a/lib/_emerge/PollScheduler.py b/lib/_emerge/PollScheduler.py index e90830885..f51c5b005 100644 --- a/lib/_emerge/PollScheduler.py +++ b/lib/_emerge/PollScheduler.py @@ -14,7 +14,7 @@ from portage.util._eventloop.global_event_loop import global_event_loop from _emerge.getloadavg import getloadavg -class PollScheduler(object): +class PollScheduler: # max time between loadavg checks (milliseconds) _loadavg_latency = None diff --git a/lib/_emerge/ProgressHandler.py b/lib/_emerge/ProgressHandler.py index f5afe6d87..d7787e040 100644 --- a/lib/_emerge/ProgressHandler.py +++ b/lib/_emerge/ProgressHandler.py @@ -2,7 +2,7 @@ # Distributed under the terms of the GNU General Public License v2 import time -class ProgressHandler(object): +class ProgressHandler: def __init__(self): self.curval = 0 self.maxval = 0 @@ -19,4 +19,3 @@ class ProgressHandler(object): def display(self): raise NotImplementedError(self) - diff --git a/lib/_emerge/RootConfig.py b/lib/_emerge/RootConfig.py index 3648d01d7..9dc9636da 100644 --- a/lib/_emerge/RootConfig.py +++ b/lib/_emerge/RootConfig.py @@ -1,7 +1,7 @@ # Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -class RootConfig(object): +class RootConfig: """This is used internally by depgraph to track information about a particular $ROOT.""" __slots__ = ("mtimedb", "root", "setconfig", "sets", "settings", "trees") diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py index 6f4564000..7906c30c8 100644 --- a/lib/_emerge/Scheduler.py +++ b/lib/_emerge/Scheduler.py @@ -1,7 +1,7 @@ # Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import division, print_function, unicode_literals +from __future__ import division, print_function from collections import deque import gc @@ -63,9 +63,6 @@ from _emerge.PackageMerge import PackageMerge from _emerge.PollScheduler import PollScheduler from _emerge.SequentialTaskQueue import SequentialTaskQueue -if sys.hexversion >= 0x3000000: - basestring = str - # enums FAILURE = 1 @@ -123,7 +120,7 @@ class Scheduler(PollScheduler): __slots__ = ("build_dir", "build_log", "pkg", "postinst_failure", "returncode") - class _ConfigPool(object): + class _ConfigPool: """Interface for a task to temporarily allocate a config instance from a pool. This allows a task to be constructed long before the config instance actually becomes needed, like @@ -1141,7 +1138,7 @@ class Scheduler(PollScheduler): if phase not in logentries: continue for msgtype, msgcontent in logentries[phase]: - if isinstance(msgcontent, basestring): + if isinstance(msgcontent, str): msgcontent = [msgcontent] for line in msgcontent: printer.eerror(line.strip("\n")) diff --git a/lib/_emerge/SequentialTaskQueue.py b/lib/_emerge/SequentialTaskQueue.py index d2551b1c6..40590b76c 100644 --- a/lib/_emerge/SequentialTaskQueue.py +++ b/lib/_emerge/SequentialTaskQueue.py @@ -2,7 +2,6 @@ # Distributed under the terms of the GNU General Public License v2 from collections import deque -import sys from portage.util.futures import asyncio from portage.util.futures.compat_coroutine import coroutine @@ -85,8 +84,5 @@ class SequentialTaskQueue(SlotObject): def __bool__(self): return bool(self._task_queue or self.running_tasks) - if sys.hexversion < 0x3000000: - __nonzero__ = __bool__ - def __len__(self): return len(self._task_queue) + len(self.running_tasks) diff --git a/lib/_emerge/SetArg.py b/lib/_emerge/SetArg.py index 5c8297547..2fee86c49 100644 --- a/lib/_emerge/SetArg.py +++ b/lib/_emerge/SetArg.py @@ -11,4 +11,3 @@ class SetArg(DependencyArg): DependencyArg.__init__(self, **kwargs) self.pset = pset self.name = self.arg[len(SETPREFIX):] - diff --git a/lib/_emerge/SpawnProcess.py b/lib/_emerge/SpawnProcess.py index 395d66bb9..91f5ed1a8 100644 --- a/lib/_emerge/SpawnProcess.py +++ b/lib/_emerge/SpawnProcess.py @@ -1,12 +1,6 @@ -# Copyright 2008-2018 Gentoo Foundation +# Copyright 2008-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -try: - import fcntl -except ImportError: - # http://bugs.jython.org/issue1074 - fcntl = None - import errno import logging import signal @@ -19,7 +13,10 @@ from portage.const import BASH_BINARY from portage.localization import _ from portage.output import EOutput from portage.util import writemsg_level +from portage.util._async.BuildLogger import BuildLogger from portage.util._async.PipeLogger import PipeLogger +from portage.util.futures import asyncio +from portage.util.futures.compat_coroutine import coroutine class SpawnProcess(SubProcess): @@ -34,8 +31,8 @@ class SpawnProcess(SubProcess): "path_lookup", "pre_exec", "close_fds", "cgroup", "unshare_ipc", "unshare_mount", "unshare_pid", "unshare_net") - __slots__ = ("args",) + \ - _spawn_kwarg_names + ("_pipe_logger", "_selinux_type",) + __slots__ = ("args", "log_filter_file") + \ + _spawn_kwarg_names + ("_main_task", "_selinux_type",) # Max number of attempts to kill the processes listed in cgroup.procs, # given that processes may fork before they can be killed. @@ -126,24 +123,55 @@ class SpawnProcess(SubProcess): stdout_fd = None if can_log and not self.background: stdout_fd = os.dup(fd_pipes_orig[1]) - # FD_CLOEXEC is enabled by default in Python >=3.4. - if sys.hexversion < 0x3040000 and fcntl is not None: - try: - fcntl.FD_CLOEXEC - except AttributeError: - pass - else: - fcntl.fcntl(stdout_fd, fcntl.F_SETFD, - fcntl.fcntl(stdout_fd, - fcntl.F_GETFD) | fcntl.FD_CLOEXEC) - self._pipe_logger = PipeLogger(background=self.background, + build_logger = BuildLogger(env=self.env, + log_path=log_file_path, + log_filter_file=self.log_filter_file, + scheduler=self.scheduler) + build_logger.start() + + pipe_logger = PipeLogger(background=self.background, scheduler=self.scheduler, input_fd=master_fd, - log_file_path=log_file_path, + log_file_path=build_logger.stdin, stdout_fd=stdout_fd) - self._pipe_logger.addExitListener(self._pipe_logger_exit) - self._pipe_logger.start() + + pipe_logger.start() + self._registered = True + self._main_task = asyncio.ensure_future(self._main(build_logger, pipe_logger), loop=self.scheduler) + self._main_task.add_done_callback(self._main_exit) + + @coroutine + def _main(self, build_logger, pipe_logger): + try: + if pipe_logger.poll() is None: + yield pipe_logger.async_wait() + if build_logger.poll() is None: + yield build_logger.async_wait() + except asyncio.CancelledError: + if pipe_logger.poll() is None: + pipe_logger.cancel() + if build_logger.poll() is None: + build_logger.cancel() + raise + + def _main_exit(self, main_task): + self._main_task = None + try: + main_task.result() + except asyncio.CancelledError: + self.cancel() + self._async_waitpid() + + def _async_wait(self): + # Allow _main_task to exit normally rather than via cancellation. + if self._main_task is None: + super(SpawnProcess, self)._async_wait() + + def _async_waitpid(self): + # Allow _main_task to exit normally rather than via cancellation. + if self._main_task is None: + super(SpawnProcess, self)._async_waitpid() def _can_log(self, slave_fd): return True @@ -167,20 +195,17 @@ class SpawnProcess(SubProcess): return spawn_func(args, **kwargs) - def _pipe_logger_exit(self, pipe_logger): - self._pipe_logger = None - self._async_waitpid() - def _unregister(self): SubProcess._unregister(self) if self.cgroup is not None: self._cgroup_cleanup() self.cgroup = None - if self._pipe_logger is not None: - self._pipe_logger.cancel() - self._pipe_logger = None + if self._main_task is not None: + self._main_task.done() or self._main_task.cancel() def _cancel(self): + if self._main_task is not None: + self._main_task.done() or self._main_task.cancel() SubProcess._cancel(self) self._cgroup_cleanup() diff --git a/lib/_emerge/SubProcess.py b/lib/_emerge/SubProcess.py index e834cb7d3..f7353926c 100644 --- a/lib/_emerge/SubProcess.py +++ b/lib/_emerge/SubProcess.py @@ -48,9 +48,12 @@ class SubProcess(AbstractPollTask): def _async_waitpid(self): """ Wait for exit status of self.pid asynchronously, and then - set the returncode and notify exit listeners. This is - prefered over _waitpid_loop, since the synchronous nature - of _waitpid_loop can cause event loop recursion. + set the returncode, and finally notify exit listeners via the + _async_wait method. Subclasses may override this method in order + to implement an alternative means to retrieve pid exit status, + or as a means to delay action until some pending task(s) have + completed (such as reading data that the subprocess is supposed + to have written to a pipe). """ if self.returncode is not None: self._async_wait() diff --git a/lib/_emerge/TaskSequence.py b/lib/_emerge/TaskSequence.py index 1f2ba94c2..8d1b0b859 100644 --- a/lib/_emerge/TaskSequence.py +++ b/lib/_emerge/TaskSequence.py @@ -1,7 +1,6 @@ -# Copyright 1999-2014 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -import sys from collections import deque from portage import os @@ -54,8 +53,5 @@ class TaskSequence(CompositeTask): def __bool__(self): return bool(self._task_queue) - if sys.hexversion < 0x3000000: - __nonzero__ = __bool__ - def __len__(self): return len(self._task_queue) diff --git a/lib/_emerge/UnmergeDepPriority.py b/lib/_emerge/UnmergeDepPriority.py index ec44a67a1..ed831ed45 100644 --- a/lib/_emerge/UnmergeDepPriority.py +++ b/lib/_emerge/UnmergeDepPriority.py @@ -43,4 +43,3 @@ class UnmergeDepPriority(AbstractDepPriority): if myvalue > self.SOFT: return "hard" return "soft" - diff --git a/lib/_emerge/UseFlagDisplay.py b/lib/_emerge/UseFlagDisplay.py index 12820e9d1..9496693c8 100644 --- a/lib/_emerge/UseFlagDisplay.py +++ b/lib/_emerge/UseFlagDisplay.py @@ -1,18 +1,15 @@ -# Copyright 1999-2013 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - import collections from itertools import chain -import sys from portage import _encodings, _unicode_encode from portage.output import red from portage.util import cmp_sort_key from portage.output import blue -class UseFlagDisplay(object): +class UseFlagDisplay: __slots__ = ('name', 'enabled', 'forced') @@ -32,14 +29,6 @@ class UseFlagDisplay(object): s = '(%s)' % s return s - if sys.hexversion < 0x3000000: - - __unicode__ = __str__ - - def __str__(self): - return _unicode_encode(self.__unicode__(), - encoding=_encodings['content']) - def _cmp_combined(a, b): """ Sort by name, combining enabled and disabled flags. diff --git a/lib/_emerge/UserQuery.py b/lib/_emerge/UserQuery.py index e20bbc6c3..d5273e4ee 100644 --- a/lib/_emerge/UserQuery.py +++ b/lib/_emerge/UserQuery.py @@ -1,7 +1,7 @@ -# Copyright 1999-2016 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function, unicode_literals +from __future__ import print_function import signal import sys @@ -10,7 +10,7 @@ from portage import _unicode_decode from portage.output import bold, create_color_func -class UserQuery(object): +class UserQuery: """The UserQuery class is used to prompt the user with a set of responses, as well as accepting and handling the responses.""" @@ -54,17 +54,12 @@ class UserQuery(object): print(bold(prompt), end=' ') try: while True: - if sys.hexversion >= 0x3000000: - try: - response = input("[%s] " % - "/".join([colours[i](responses[i]) - for i in range(len(responses))])) - except UnicodeDecodeError as e: - response = _unicode_decode(e.object).rstrip('\n') - else: - response=raw_input("["+"/".join([colours[i](responses[i]) - for i in range(len(responses))])+"] ") - response = _unicode_decode(response) + try: + response = input("[%s] " % + "/".join([colours[i](responses[i]) + for i in range(len(responses))])) + except UnicodeDecodeError as e: + response = _unicode_decode(e.object).rstrip('\n') if response or not enter_invalid: for key in responses: # An empty response will match the diff --git a/lib/_emerge/_find_deep_system_runtime_deps.py b/lib/_emerge/_find_deep_system_runtime_deps.py index ca09d83ac..d88296d5f 100644 --- a/lib/_emerge/_find_deep_system_runtime_deps.py +++ b/lib/_emerge/_find_deep_system_runtime_deps.py @@ -35,4 +35,3 @@ def _find_deep_system_runtime_deps(graph): node_stack.append(child) return deep_system_deps - diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py index d8e9d7774..961ecf3d1 100644 --- a/lib/_emerge/actions.py +++ b/lib/_emerge/actions.py @@ -1,19 +1,15 @@ # Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import division, print_function, unicode_literals +from __future__ import division, print_function import collections import errno import logging import operator import platform -import pwd -import random import re import signal -import socket -import stat import subprocess import sys import tempfile @@ -31,7 +27,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.util.locale:check_locale', 'portage.emaint.modules.sync.sync:SyncRepos', '_emerge.chk_updated_cfg_files:chk_updated_cfg_files', - '_emerge.help:help@emerge_help', + '_emerge.help:emerge_help', '_emerge.post_emerge:display_news_notification,post_emerge', '_emerge.stdout_spinner:stdout_spinner', ) @@ -95,11 +91,6 @@ from _emerge.UnmergeDepPriority import UnmergeDepPriority from _emerge.UseFlagDisplay import pkg_use_display from _emerge.UserQuery import UserQuery -if sys.hexversion >= 0x3000000: - long = int - _unicode = str -else: - _unicode = unicode def action_build(emerge_config, trees=DeprecationWarning, mtimedb=DeprecationWarning, myopts=DeprecationWarning, @@ -855,7 +846,7 @@ def _calc_depclean(settings, trees, ldpath_mtimes, protected_set.add("=" + pkg.cpv) continue except portage.exception.InvalidDependString as e: - show_invalid_depstring_notice(pkg, _unicode(e)) + show_invalid_depstring_notice(pkg, str(e)) del e protected_set.add("=" + pkg.cpv) continue @@ -908,7 +899,7 @@ def _calc_depclean(settings, trees, ldpath_mtimes, protected_set.add("=" + pkg.cpv) continue except portage.exception.InvalidDependString as e: - show_invalid_depstring_notice(pkg, _unicode(e)) + show_invalid_depstring_notice(pkg, str(e)) del e protected_set.add("=" + pkg.cpv) continue @@ -925,7 +916,7 @@ def _calc_depclean(settings, trees, ldpath_mtimes, if excluded_set.findAtomForPackage(pkg): required_sets['__excluded__'].add("=" + pkg.cpv) except portage.exception.InvalidDependString as e: - show_invalid_depstring_notice(pkg, _unicode(e)) + show_invalid_depstring_notice(pkg, str(e)) del e required_sets['__excluded__'].add("=" + pkg.cpv) @@ -990,14 +981,14 @@ def _calc_depclean(settings, trees, ldpath_mtimes, # visible in the unevaluated form of the atom. In this # case, we must display the unevaluated atom, so that # the user can see the conditional USE deps that would - # otherwise be invisible. Use Atom(_unicode(atom)) to + # otherwise be invisible. Use Atom(str(atom)) to # test for a package where this case would matter. This # is not necessarily the same as atom.without_use, - # since Atom(_unicode(atom)) may still contain some + # since Atom(str(atom)) may still contain some # USE dependencies that remain after evaluation of # conditionals. if atom.package and atom != atom.unevaluated_atom and \ - vardb.match(Atom(_unicode(atom))): + vardb.match(Atom(str(atom))): msg.append(" %s (%s) pulled in by:" % (atom.unevaluated_atom, atom)) else: @@ -1069,7 +1060,7 @@ def _calc_depclean(settings, trees, ldpath_mtimes, key=operator.attrgetter('package')) parent_strs.append("%s requires %s" % (getattr(parent, "cpv", parent), - ", ".join(_unicode(atom) for atom in atoms))) + ", ".join(str(atom) for atom in atoms))) parent_strs.sort() msg = [] msg.append(" %s pulled in by:\n" % (child_node.cpv,)) @@ -1082,10 +1073,9 @@ def _calc_depclean(settings, trees, ldpath_mtimes, """Sort Package instances by cpv.""" if pkg1.cpv > pkg2.cpv: return 1 - elif pkg1.cpv == pkg2.cpv: + if pkg1.cpv == pkg2.cpv: return 0 - else: - return -1 + return -1 def create_cleanlist(): @@ -1542,7 +1532,7 @@ def action_deselect(settings, trees, opts, atoms): writemsg_stdout( ">>> %s %s from \"%s\" favorites file...\n" % - (action_desc, colorize("INFORM", _unicode(atom)), + (action_desc, colorize("INFORM", str(atom)), filename), noiselevel=-1) if '--ask' in opts: @@ -1563,7 +1553,7 @@ def action_deselect(settings, trees, opts, atoms): world_set.unlock() return os.EX_OK -class _info_pkgs_ver(object): +class _info_pkgs_ver: def __init__(self, ver, repo_suffix, provide_suffix): self.ver = ver self.repo_suffix = repo_suffix @@ -3031,7 +3021,7 @@ def run_action(emerge_config): emerge_config.target_config.trees['vartree'].dbapi) + '\n', noiselevel=-1) return 0 - elif emerge_config.action == 'help': + if emerge_config.action == 'help': emerge_help() return 0 @@ -3065,7 +3055,7 @@ def run_action(emerge_config): writemsg_stdout("".join("%s\n" % s for s in sorted(emerge_config.target_config.sets))) return os.EX_OK - elif emerge_config.action == "check-news": + if emerge_config.action == "check-news": news_counts = count_unread_news( emerge_config.target_config.trees["porttree"].dbapi, emerge_config.target_config.trees["vartree"].dbapi) @@ -3237,8 +3227,6 @@ def run_action(emerge_config): if not "--pretend" in emerge_config.opts: time_fmt = "%b %d, %Y %H:%M:%S" - if sys.hexversion < 0x3000000: - time_fmt = portage._unicode_encode(time_fmt) time_str = time.strftime(time_fmt, time.localtime(time.time())) # Avoid potential UnicodeDecodeError in Python 2, since strftime # returns bytes in Python 2, and %b may contain non-ascii chars. @@ -3290,7 +3278,7 @@ def run_action(emerge_config): if "sync" == emerge_config.action: return action_sync(emerge_config) - elif "metadata" == emerge_config.action: + if "metadata" == emerge_config.action: action_metadata(emerge_config.target_config.settings, emerge_config.target_config.trees['porttree'].dbapi, emerge_config.opts) diff --git a/lib/_emerge/create_depgraph_params.py b/lib/_emerge/create_depgraph_params.py index 81edcb9c0..0d0e07b9c 100644 --- a/lib/_emerge/create_depgraph_params.py +++ b/lib/_emerge/create_depgraph_params.py @@ -195,4 +195,3 @@ def create_depgraph_params(myopts, myaction): noiselevel=-1, level=logging.DEBUG) return myparams - diff --git a/lib/_emerge/create_world_atom.py b/lib/_emerge/create_world_atom.py index a7f3e30bf..8b506c36e 100644 --- a/lib/_emerge/create_world_atom.py +++ b/lib/_emerge/create_world_atom.py @@ -1,15 +1,10 @@ # Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -import sys from portage.dep import Atom, _repo_separator from portage.exception import InvalidData -if sys.hexversion >= 0x3000000: - _unicode = str -else: - _unicode = unicode def create_world_atom(pkg, args_set, root_config, before_install=False): """Create a new atom for the world file if one does not exist. If the @@ -43,7 +38,7 @@ def create_world_atom(pkg, args_set, root_config, before_install=False): for cpv in portdb.match(Atom(cp)): for repo in repos: try: - available_slots.add(portdb._pkg_str(_unicode(cpv), repo).slot) + available_slots.add(portdb._pkg_str(str(cpv), repo).slot) except (KeyError, InvalidData): pass @@ -98,7 +93,7 @@ def create_world_atom(pkg, args_set, root_config, before_install=False): for repo in repos: try: matched_slots.add( - portdb._pkg_str(_unicode(cpv), repo).slot) + portdb._pkg_str(str(cpv), repo).slot) except (KeyError, InvalidData): pass @@ -126,4 +121,3 @@ def create_world_atom(pkg, args_set, root_config, before_install=False): providers[0].cp == arg_atom.cp: return None return new_world_atom - diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index 13c2b658f..d2406aeec 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -1,7 +1,7 @@ # Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import division, print_function, unicode_literals +from __future__ import division, print_function import collections import errno @@ -9,7 +9,6 @@ import functools import io import logging import stat -import sys import textwrap import warnings from collections import deque, OrderedDict @@ -88,13 +87,6 @@ from _emerge.resolver.slot_collision import slot_conflict_handler from _emerge.resolver.circular_dependency import circular_dependency_handler from _emerge.resolver.output import Display, format_unmatched_atom -if sys.hexversion >= 0x3000000: - basestring = str - long = int - _unicode = str -else: - _unicode = unicode - # Exposes a depgraph interface to dep_check. _dep_check_graph_interface = collections.namedtuple('_dep_check_graph_interface',( # Indicates a removal action, like depclean or prune. @@ -103,7 +95,7 @@ _dep_check_graph_interface = collections.namedtuple('_dep_check_graph_interface' 'want_update_pkg', )) -class _scheduler_graph_config(object): +class _scheduler_graph_config: def __init__(self, trees, pkg_cache, graph, mergelist): self.trees = trees self.pkg_cache = pkg_cache @@ -120,7 +112,7 @@ def _wildcard_set(atoms): pkgs.add(x) return pkgs -class _frozen_depgraph_config(object): +class _frozen_depgraph_config: def __init__(self, settings, trees, myopts, params, spinner): self.settings = settings @@ -194,7 +186,7 @@ class _frozen_depgraph_config(object): self.rebuild_if_new_ver = "--rebuild-if-new-ver" in myopts self.rebuild_if_unbuilt = "--rebuild-if-unbuilt" in myopts -class _depgraph_sets(object): +class _depgraph_sets: def __init__(self): # contains all sets added to the graph self.sets = {} @@ -205,7 +197,7 @@ class _depgraph_sets(object): self.atoms = InternalPackageSet(allow_repo=True) self.atom_arg_map = {} -class _rebuild_config(object): +class _rebuild_config: def __init__(self, frozen_config, backtrack_parameters): self._graph = digraph() self._frozen_config = frozen_config @@ -270,7 +262,7 @@ class _rebuild_config(object): if self._needs_rebuild(dep_pkg): self.rebuild_list.add(root_slot) return True - elif ("--usepkg" in self._frozen_config.myopts and + if ("--usepkg" in self._frozen_config.myopts and (dep_root_slot in self.reinstall_list or dep_root_slot in self.rebuild_list or not dep_pkg.installed)): @@ -304,14 +296,14 @@ class _rebuild_config(object): # built without dep_pkg. Force rebuild. self.rebuild_list.add(root_slot) return True - elif (parent.installed and + if (parent.installed and root_slot not in self.reinstall_list): try: bin_build_time, = bindb.aux_get(parent.cpv, ["BUILD_TIME"]) except KeyError: continue - if bin_build_time != _unicode(parent.build_time): + if bin_build_time != str(parent.build_time): # 2) Remote binary package is valid, and local package # is not up to date. Force reinstall. reinstall = True @@ -374,7 +366,7 @@ class _use_changes(tuple): return obj -class _dynamic_depgraph_config(object): +class _dynamic_depgraph_config: """ ``dynamic_depgraph_config`` is an object that is used to collect settings and important data structures that are @@ -598,7 +590,7 @@ class _dynamic_depgraph_config(object): dbs.append((vardb, "installed", True, True, db_keys)) self._filtered_trees[myroot]["dbs"] = dbs -class depgraph(object): +class depgraph: # Represents the depth of a node that is unreachable from explicit # user arguments (or their deep dependencies). Such nodes are pulled @@ -710,7 +702,7 @@ class depgraph(object): self._dynamic_deps_proc_exit(pkg, fake_vartree)) yield proc - class _dynamic_deps_proc_exit(object): + class _dynamic_deps_proc_exit: __slots__ = ('_pkg', '_fake_vartree') @@ -2574,14 +2566,14 @@ class depgraph(object): Atom("=%s" % inst_pkg.cpv)): if not pkg.built: return pkg.slot_atom - elif not pkg.installed: + if not pkg.installed: # avoid using SLOT from a built instance built_pkgs.append(pkg) for pkg in self._iter_similar_available(inst_pkg, inst_pkg.slot_atom): if not pkg.built: return pkg.slot_atom - elif not pkg.installed: + if not pkg.installed: # avoid using SLOT from a built instance built_pkgs.append(pkg) @@ -2907,58 +2899,58 @@ class depgraph(object): self._slot_operator_unsatisfied_probe(dep)): self._slot_operator_unsatisfied_backtrack(dep) return 1 - else: - # This is for backward-compatibility with previous - # behavior, so that installed packages with unsatisfied - # dependencies trigger an error message but do not - # cause the dependency calculation to fail. Only do - # this if the parent is already in the runtime package - # mask, since otherwise we need to backtrack. - if (dep.parent.installed and - dep.parent in self._dynamic_config._runtime_pkg_mask and - not any(self._iter_match_pkgs_any( - dep.parent.root_config, dep.atom))): - self._dynamic_config._initially_unsatisfied_deps.append(dep) - return 1 - - # Do not backtrack if only USE have to be changed in - # order to satisfy the dependency. Note that when - # want_restart_for_use_change sets the need_restart - # flag, it causes _select_pkg_highest_available to - # return None, and eventually we come through here - # and skip the "missing dependency" backtracking path. - dep_pkg, existing_node = \ - self._select_package(dep.root, - dep.atom.without_use if dep.atom.package - else dep.atom, onlydeps=dep.onlydeps) - if dep_pkg is None: - - # In order to suppress the sort of aggressive - # backtracking that can trigger undesirable downgrades - # as in bug 693836, do not backtrack if there's an - # available package which was involved in a slot - # conflict and satisfied all involved parent atoms. - for dep_pkg, reasons in self._dynamic_config._runtime_pkg_mask.items(): - if (dep.atom.match(dep_pkg) and - len(reasons) == 1 and - not reasons.get("slot conflict", True)): - self._dynamic_config._skip_restart = True - return 0 - self._dynamic_config._backtrack_infos["missing dependency"] = dep - self._dynamic_config._need_restart = True - if debug: - msg = [] - msg.append("") - msg.append("") - msg.append("backtracking due to unsatisfied dep:") - msg.append(" parent: %s" % dep.parent) - msg.append(" priority: %s" % dep.priority) - msg.append(" root: %s" % dep.root) - msg.append(" atom: %s" % dep.atom) - msg.append("") - writemsg_level("".join("%s\n" % l for l in msg), - noiselevel=-1, level=logging.DEBUG) + # This is for backward-compatibility with previous + # behavior, so that installed packages with unsatisfied + # dependencies trigger an error message but do not + # cause the dependency calculation to fail. Only do + # this if the parent is already in the runtime package + # mask, since otherwise we need to backtrack. + if (dep.parent.installed and + dep.parent in self._dynamic_config._runtime_pkg_mask and + not any(self._iter_match_pkgs_any( + dep.parent.root_config, dep.atom))): + self._dynamic_config._initially_unsatisfied_deps.append(dep) + return 1 + + # Do not backtrack if only USE have to be changed in + # order to satisfy the dependency. Note that when + # want_restart_for_use_change sets the need_restart + # flag, it causes _select_pkg_highest_available to + # return None, and eventually we come through here + # and skip the "missing dependency" backtracking path. + dep_pkg, existing_node = \ + self._select_package(dep.root, + dep.atom.without_use if dep.atom.package + else dep.atom, onlydeps=dep.onlydeps) + if dep_pkg is None: + + # In order to suppress the sort of aggressive + # backtracking that can trigger undesirable downgrades + # as in bug 693836, do not backtrack if there's an + # available package which was involved in a slot + # conflict and satisfied all involved parent atoms. + for dep_pkg, reasons in self._dynamic_config._runtime_pkg_mask.items(): + if (dep.atom.match(dep_pkg) and + len(reasons) == 1 and + not reasons.get("slot conflict", True)): + self._dynamic_config._skip_restart = True + return 0 + + self._dynamic_config._backtrack_infos["missing dependency"] = dep + self._dynamic_config._need_restart = True + if debug: + msg = [] + msg.append("") + msg.append("") + msg.append("backtracking due to unsatisfied dep:") + msg.append(" parent: %s" % dep.parent) + msg.append(" priority: %s" % dep.priority) + msg.append(" root: %s" % dep.root) + msg.append(" atom: %s" % dep.atom) + msg.append("") + writemsg_level("".join("%s\n" % l for l in msg), + noiselevel=-1, level=logging.DEBUG) return 0 @@ -3198,7 +3190,7 @@ class depgraph(object): dep_stack = self._dynamic_config._dep_stack if "recurse" not in self._dynamic_config.myparams: return 1 - elif pkg.installed and not recurse: + if pkg.installed and not recurse: dep_stack = self._dynamic_config._ignored_deps self._spinner_update() @@ -5664,9 +5656,8 @@ class depgraph(object): if atom.package: return self._iter_match_pkgs_atom(root_config, pkg_type, atom, onlydeps=onlydeps) - else: - return self._iter_match_pkgs_soname(root_config, pkg_type, - atom, onlydeps=onlydeps) + return self._iter_match_pkgs_soname(root_config, pkg_type, + atom, onlydeps=onlydeps) def _iter_match_pkgs_soname(self, root_config, pkg_type, atom, onlydeps=False): @@ -5761,7 +5752,7 @@ class depgraph(object): other_installed, other_keys in dbs: try: if portage.dep._match_slot(atom, - other_db._pkg_str(_unicode(cpv), None)): + other_db._pkg_str(str(cpv), None)): slot_available = True break except (KeyError, InvalidData): @@ -5881,12 +5872,11 @@ class depgraph(object): deep = self._dynamic_config.myparams.get("deep", 0) if depth is self._UNREACHABLE_DEPTH: return True - elif deep is True: + if deep is True: return False - else: - # All non-integer cases are handled above, - # so both values must be int type. - return depth > deep + # All non-integer cases are handled above, + # so both values must be int type. + return depth > deep def _depth_increment(self, depth, n=1): """ @@ -5941,7 +5931,7 @@ class depgraph(object): return build_time == inst_pkg.build_time - class _AutounmaskLevel(object): + class _AutounmaskLevel: __slots__ = ("allow_use_changes", "allow_unstable_keywords", "allow_license_changes", \ "allow_missing_keywords", "allow_unmasks") @@ -6169,8 +6159,7 @@ class depgraph(object): if target_use is None: if needed_use_config_change is None: return pkg.use.enabled - else: - return needed_use_config_change[0] + return needed_use_config_change[0] if needed_use_config_change is not None: old_use = needed_use_config_change[0] @@ -7484,7 +7473,7 @@ class depgraph(object): mygraph.order.sort(key=cmp_sort_key(cmp_merge_preference)) - def altlist(self, reversed=DeprecationWarning): + def altlist(self, reversed=DeprecationWarning): # pylint: disable=redefined-builtin if reversed is not DeprecationWarning: warnings.warn("The reversed parameter of " @@ -9263,7 +9252,7 @@ class depgraph(object): filename = "world" writemsg_stdout( ">>> Recording %s in \"%s\" favorites file...\n" % - (colorize("INFORM", _unicode(a)), filename), noiselevel=-1) + (colorize("INFORM", str(a)), filename), noiselevel=-1) world_set.update(all_added) if world_locked: @@ -9458,7 +9447,7 @@ class depgraph(object): depgraph_sets = self._dynamic_config.sets[root_config.root] args = [] for x in favorites: - if not isinstance(x, basestring): + if not isinstance(x, str): continue if x in ("system", "world"): x = SETPREFIX + x @@ -9755,7 +9744,7 @@ class _dep_check_composite_db(dbapi): if not avoid_update: if not use_ebuild_visibility and usepkgonly: return False - elif not self._depgraph._equiv_ebuild_visible(pkg): + if not self._depgraph._equiv_ebuild_visible(pkg): return False if pkg.cp.startswith("virtual/"): diff --git a/lib/_emerge/emergelog.py b/lib/_emerge/emergelog.py index cb7bf77b3..3562f8eb3 100644 --- a/lib/_emerge/emergelog.py +++ b/lib/_emerge/emergelog.py @@ -1,10 +1,7 @@ # Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - import io -import sys import time import portage from portage import os diff --git a/lib/_emerge/help.py b/lib/_emerge/help.py index 2ccd323aa..de3d7b593 100644 --- a/lib/_emerge/help.py +++ b/lib/_emerge/help.py @@ -5,7 +5,7 @@ from __future__ import print_function from portage.output import bold, turquoise, green -def help(): +def emerge_help(): print(bold("emerge:")+" command-line interface to the Portage system") print(bold("Usage:")) print(" "+turquoise("emerge")+" [ "+green("options")+" ] [ "+green("action")+" ] [ "+turquoise("ebuild")+" | "+turquoise("tbz2")+" | "+turquoise("file")+" | "+turquoise("@set")+" | "+turquoise("atom")+" ] [ ... ]") diff --git a/lib/_emerge/main.py b/lib/_emerge/main.py index 95855ef2d..12323b17e 100644 --- a/lib/_emerge/main.py +++ b/lib/_emerge/main.py @@ -1,4 +1,4 @@ -# Copyright 1999-2019 Gentoo Authors +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -16,14 +16,12 @@ portage.proxy.lazyimport.lazyimport(globals(), 'textwrap', '_emerge.actions:load_emerge_config,run_action,' + \ 'validate_ebuild_environment', - '_emerge.help:help@emerge_help', + '_emerge.help:emerge_help', '_emerge.is_valid_package_atom:insert_category_into_atom' ) from portage import os from portage.sync import _SUBMODULE_PATH_MAP -if sys.hexversion >= 0x3000000: - long = int options=[ "--alphabetical", @@ -101,7 +99,7 @@ def insert_optional_args(args): this feature natively. """ - class valid_integers(object): + class valid_integers: def __contains__(self, s): try: return int(s) >= 0 @@ -110,7 +108,7 @@ def insert_optional_args(args): valid_integers = valid_integers() - class valid_floats(object): + class valid_floats: def __contains__(self, s): try: return float(s) >= 0 @@ -1241,10 +1239,10 @@ def emerge_main(args=None): if myaction == "help": emerge_help() return os.EX_OK - elif myaction == "moo": + if myaction == "moo": print(COWSAY_MOO % platform.system()) return os.EX_OK - elif myaction == "sync": + if myaction == "sync": # need to set this to True now in order for the repository config # loading to allow new repos with non-existent directories portage._sync_mode = True diff --git a/lib/_emerge/resolver/DbapiProvidesIndex.py b/lib/_emerge/resolver/DbapiProvidesIndex.py index 1650edd4e..8c7c05f06 100644 --- a/lib/_emerge/resolver/DbapiProvidesIndex.py +++ b/lib/_emerge/resolver/DbapiProvidesIndex.py @@ -1,11 +1,10 @@ -# Copyright 2015 Gentoo Foundation +# Copyright 2015-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import bisect import collections -import sys -class DbapiProvidesIndex(object): +class DbapiProvidesIndex: """ The DbapiProvidesIndex class is used to wrap existing dbapi interfaces, index packages by the sonames that they provide, and @@ -73,9 +72,6 @@ class PackageDbapiProvidesIndex(DbapiProvidesIndex): def __bool__(self): return bool(self._db) - if sys.hexversion < 0x3000000: - __nonzero__ = __bool__ - def __iter__(self): return iter(self._db) diff --git a/lib/_emerge/resolver/backtracking.py b/lib/_emerge/resolver/backtracking.py index 430c81186..bc3fb3206 100644 --- a/lib/_emerge/resolver/backtracking.py +++ b/lib/_emerge/resolver/backtracking.py @@ -3,7 +3,7 @@ import copy -class BacktrackParameter(object): +class BacktrackParameter: __slots__ = ( "circular_dependency", @@ -65,7 +65,7 @@ class BacktrackParameter(object): self.prune_rebuilds == other.prune_rebuilds -class _BacktrackNode(object): +class _BacktrackNode: __slots__ = ( "parameter", "depth", "mask_steps", "terminal", @@ -81,7 +81,7 @@ class _BacktrackNode(object): return self.parameter == other.parameter -class Backtracker(object): +class Backtracker: __slots__ = ( "_max_depth", "_unexplored_nodes", "_current_node", "_nodes", "_root", @@ -119,8 +119,7 @@ class Backtracker(object): node = self._unexplored_nodes.pop() self._current_node = node return copy.deepcopy(node.parameter) - else: - return None + return None def __len__(self): diff --git a/lib/_emerge/resolver/circular_dependency.py b/lib/_emerge/resolver/circular_dependency.py index 5c119567b..d0c026dad 100644 --- a/lib/_emerge/resolver/circular_dependency.py +++ b/lib/_emerge/resolver/circular_dependency.py @@ -1,7 +1,7 @@ # Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function, unicode_literals +from __future__ import print_function from itertools import chain, product import logging @@ -13,7 +13,7 @@ from portage.util import writemsg_level from _emerge.DepPrioritySatisfiedRange import DepPrioritySatisfiedRange from _emerge.Package import Package -class circular_dependency_handler(object): +class circular_dependency_handler: MAX_AFFECTING_USE = 10 diff --git a/lib/_emerge/resolver/output.py b/lib/_emerge/resolver/output.py index ed88cc51f..9483898de 100644 --- a/lib/_emerge/resolver/output.py +++ b/lib/_emerge/resolver/output.py @@ -1,16 +1,13 @@ -# Copyright 2010-2019 Gentoo Authors +# Copyright 2010-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 """Resolver output display operation. """ -from __future__ import unicode_literals - __all__ = ( "Display", "format_unmatched_atom", ) -import sys import portage from portage import os @@ -34,13 +31,7 @@ from _emerge.resolver.output_helpers import ( _DisplayConfig, _tree_display, _PackageCounters, _create_use_string, _calc_changelog, PkgInfo) from _emerge.show_invalid_depstring_notice import show_invalid_depstring_notice -if sys.hexversion >= 0x3000000: - basestring = str - _unicode = str -else: - _unicode = unicode - -class Display(object): +class Display: """Formats and outputs the depgrah supplied it for merge/re-merge, etc. __call__() @@ -88,19 +79,19 @@ class Display(object): addl = "%s " % (colorize(self.blocker_style, "B"),) addl += self.empty_space_in_brackets() self.resolved = dep_expand( - _unicode(blocker.atom).lstrip("!"), mydb=self.vardb, + str(blocker.atom).lstrip("!"), mydb=self.vardb, settings=self.pkgsettings ) if self.conf.columns and self.conf.quiet: - addl += " " + colorize(self.blocker_style, _unicode(self.resolved)) + addl += " " + colorize(self.blocker_style, str(self.resolved)) else: addl = "[%s %s] %s%s" % \ (colorize(self.blocker_style, "blocks"), addl, self.indent, - colorize(self.blocker_style, _unicode(self.resolved)) + colorize(self.blocker_style, str(self.resolved)) ) block_parents = self.conf.blocker_parents.parent_nodes(blocker) - block_parents = set(_unicode(pnode.cpv) for pnode in block_parents) + block_parents = set(str(pnode.cpv) for pnode in block_parents) block_parents = ", ".join(block_parents) if blocker.atom.blocker.overlap.forbid: blocking_desc = "hard blocking" @@ -109,7 +100,7 @@ class Display(object): if self.resolved != blocker.atom: addl += colorize(self.blocker_style, " (\"%s\" is %s %s)" % - (_unicode(blocker.atom).lstrip("!"), + (str(blocker.atom).lstrip("!"), blocking_desc, block_parents)) else: addl += colorize(self.blocker_style, @@ -269,26 +260,24 @@ class Display(object): if pkg_info.built: if pkg_info.system: return colorize("PKG_BINARY_MERGE_SYSTEM", pkg_str) - elif pkg_info.world: + if pkg_info.world: return colorize("PKG_BINARY_MERGE_WORLD", pkg_str) - else: - return colorize("PKG_BINARY_MERGE", pkg_str) - else: - if pkg_info.system: - return colorize("PKG_MERGE_SYSTEM", pkg_str) - elif pkg_info.world: - return colorize("PKG_MERGE_WORLD", pkg_str) - else: - return colorize("PKG_MERGE", pkg_str) - elif pkg_info.operation == "uninstall": - return colorize("PKG_UNINSTALL", pkg_str) - else: + return colorize("PKG_BINARY_MERGE", pkg_str) + if pkg_info.system: - return colorize("PKG_NOMERGE_SYSTEM", pkg_str) - elif pkg_info.world: - return colorize("PKG_NOMERGE_WORLD", pkg_str) - else: - return colorize("PKG_NOMERGE", pkg_str) + return colorize("PKG_MERGE_SYSTEM", pkg_str) + if pkg_info.world: + return colorize("PKG_MERGE_WORLD", pkg_str) + return colorize("PKG_MERGE", pkg_str) + + if pkg_info.operation == "uninstall": + return colorize("PKG_UNINSTALL", pkg_str) + + if pkg_info.system: + return colorize("PKG_NOMERGE_SYSTEM", pkg_str) + if pkg_info.world: + return colorize("PKG_NOMERGE_WORLD", pkg_str) + return colorize("PKG_NOMERGE", pkg_str) def verbose_size(self, pkg, repoadd_set, pkg_info): @@ -316,7 +305,7 @@ class Display(object): depstr, = db.aux_get(pkg.cpv, ["SRC_URI"], myrepo=pkg.repo) show_invalid_depstring_notice( - pkg, _unicode(e)) + pkg, str(e)) raise except SignatureException: # missing/invalid binary package SIZE signature @@ -447,7 +436,7 @@ class Display(object): ver_str = self._append_slot(ver_str, pkg, pkg_info) ver_str = self._append_repository(ver_str, pkg, pkg_info) if self.conf.quiet: - myprint = _unicode(pkg_info.attr_display) + " " + self.indent + \ + myprint = str(pkg_info.attr_display) + " " + self.indent + \ self.pkgprint(pkg_info.cp, pkg_info) myprint = myprint+darkblue(" "+ver_str)+" " myprint = myprint+pkg_info.oldbest @@ -486,7 +475,7 @@ class Display(object): ver_str = self._append_slot(ver_str, pkg, pkg_info) ver_str = self._append_repository(ver_str, pkg, pkg_info) if self.conf.quiet: - myprint = _unicode(pkg_info.attr_display) + " " + self.indent + \ + myprint = str(pkg_info.attr_display) + " " + self.indent + \ self.pkgprint(pkg_info.cp, pkg_info) myprint = myprint+" "+green(ver_str)+" " myprint = myprint+pkg_info.oldbest @@ -543,7 +532,7 @@ class Display(object): @param show_repos: bool. """ for msg in self.print_msg: - if isinstance(msg, basestring): + if isinstance(msg, str): writemsg_stdout("%s\n" % (msg,), noiselevel=-1) continue myprint, self.verboseadd, repoadd = msg diff --git a/lib/_emerge/resolver/output_helpers.py b/lib/_emerge/resolver/output_helpers.py index b83717e93..25aa925b4 100644 --- a/lib/_emerge/resolver/output_helpers.py +++ b/lib/_emerge/resolver/output_helpers.py @@ -1,18 +1,15 @@ -# Copyright 2010-2014 Gentoo Foundation +# Copyright 2010-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 """Contains private support functions for the Display class in output.py """ -from __future__ import unicode_literals - __all__ = ( ) import io import re -import sys from portage import os from portage import _encodings, _unicode_encode @@ -30,11 +27,7 @@ from _emerge.Blocker import Blocker from _emerge.Package import Package -if sys.hexversion >= 0x3000000: - basestring = str - - -class _RepoDisplay(object): +class _RepoDisplay: def __init__(self, roots): self._shown_repos = {} self._unknown_repo = False @@ -82,16 +75,8 @@ class _RepoDisplay(object): " indicates that the source repository could not be determined\n") return "".join(output) - if sys.hexversion < 0x3000000: - - __unicode__ = __str__ - - def __str__(self): - return _unicode_encode(self.__unicode__(), - encoding=_encodings['content']) - -class _PackageCounters(object): +class _PackageCounters: def __init__(self): self.upgrades = 0 @@ -169,7 +154,7 @@ class _PackageCounters(object): return "".join(myoutput) -class _DisplayConfig(object): +class _DisplayConfig: def __init__(self, depgraph, mylist, favorites, verbosity): frozen_config = depgraph._frozen_config @@ -484,7 +469,7 @@ def _prune_tree_display(display_list): del display_list[i] -def _calc_changelog(ebuildpath,current,next): +def _calc_changelog(ebuildpath,current,next): # pylint: disable=redefined-builtin if ebuildpath == None or not os.path.exists(ebuildpath): return [] current = '-'.join(catpkgsplit(current)[1:]) @@ -599,7 +584,7 @@ def _find_changelog_tags(changelog): _strip_header_comments(changelog[release_end:].splitlines()))) return divs -class PkgInfo(object): +class PkgInfo: """Simple class to hold instance attributes for current information about the pkg being printed. """ @@ -683,11 +668,3 @@ class PkgAttrDisplay(SlotObject): output.append(self.mask) return "".join(output) - - if sys.hexversion < 0x3000000: - - __unicode__ = __str__ - - def __str__(self): - return _unicode_encode(self.__unicode__(), - encoding=_encodings['content']) diff --git a/lib/_emerge/resolver/package_tracker.py b/lib/_emerge/resolver/package_tracker.py index ccb0b11cf..2afa7923b 100644 --- a/lib/_emerge/resolver/package_tracker.py +++ b/lib/_emerge/resolver/package_tracker.py @@ -29,7 +29,7 @@ class PackageConflict(_PackageConflict): return len(self.pkgs) -class PackageTracker(object): +class PackageTracker: """ **Behavior** @@ -358,7 +358,7 @@ class PackageTracker(object): return self.contains(pkg, installed=True) -class PackageTrackerDbapiWrapper(object): +class PackageTrackerDbapiWrapper: """ A wrpper class that provides parts of the legacy dbapi interface. Remove it once all consumers have diff --git a/lib/_emerge/resolver/slot_collision.py b/lib/_emerge/resolver/slot_collision.py index 682a3a0a5..cc16287de 100644 --- a/lib/_emerge/resolver/slot_collision.py +++ b/lib/_emerge/resolver/slot_collision.py @@ -1,9 +1,8 @@ -# Copyright 2010-2014 Gentoo Foundation +# Copyright 2010-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function, unicode_literals +from __future__ import print_function -import sys from portage import _encodings, _unicode_encode from _emerge.AtomArg import AtomArg @@ -16,10 +15,8 @@ from portage._sets.base import InternalPackageSet from portage.util import writemsg from portage.versions import cpv_getversion, vercmp -if sys.hexversion >= 0x3000000: - basestring = str -class slot_conflict_handler(object): +class slot_conflict_handler: """This class keeps track of all slot conflicts and provides an interface to get possible solutions. @@ -374,11 +371,11 @@ class slot_conflict_handler(object): selected_for_display = set() unconditional_use_deps = set() - for (type, sub_type), parents in collision_reasons.items(): - #From each (type, sub_type) pair select at least one atom. + for (ctype, sub_type), parents in collision_reasons.items(): + #From each (ctype, sub_type) pair select at least one atom. #Try to select as few atoms as possible - if type == "version": + if ctype == "version": #Find the atom with version that is as far away as possible. best_matches = {} for ppkg, atom, other_pkg in parents: @@ -398,7 +395,7 @@ class slot_conflict_handler(object): if not verboseconflicts: selected_for_display.update( best_matches.values()) - elif type in ("soname", "slot"): + elif ctype in ("soname", "slot"): # Check for packages that might need to # be rebuilt, but cannot be rebuilt for # some reason. @@ -426,7 +423,7 @@ class slot_conflict_handler(object): selected_for_display.add((ppkg, atom)) if not verboseconflicts: break - elif type == "use": + elif ctype == "use": #Prefer atoms with unconditional use deps over, because it's #not possible to change them on the parent, which means there #are fewer possible solutions. @@ -467,7 +464,7 @@ class slot_conflict_handler(object): # If the list is long, people can simply # use a pager. selected_for_display.add((ppkg, atom)) - elif type == "AtomArg": + elif ctype == "AtomArg": for ppkg, atom in parents: selected_for_display.add((ppkg, atom)) @@ -739,9 +736,9 @@ class slot_conflict_handler(object): all_involved_flags = [] #Go through all slot conflicts - for id, pkg in enumerate(config): + for idx, pkg in enumerate(config): involved_flags = {} - for ppkg, atom in all_conflict_atoms_by_slotatom[id]: + for ppkg, atom in all_conflict_atoms_by_slotatom[idx]: if not atom.package: continue @@ -849,8 +846,8 @@ class slot_conflict_handler(object): if self.debug: writemsg("All involved flags:\n", noiselevel=-1) - for id, involved_flags in enumerate(all_involved_flags): - writemsg(" %s\n" % (config[id],), noiselevel=-1) + for idx, involved_flags in enumerate(all_involved_flags): + writemsg(" %s\n" % (config[idx],), noiselevel=-1) for flag, state in involved_flags.items(): writemsg(" " + flag + ": " + state + "\n", noiselevel=-1) @@ -1068,10 +1065,9 @@ class slot_conflict_handler(object): if is_valid_solution and required_changes: return required_changes - else: - return None + return None -class _configuration_generator(object): +class _configuration_generator: def __init__(self, conflict_pkgs): #reorder packages such that installed packages come last self.conflict_pkgs = [] @@ -1098,11 +1094,11 @@ class _configuration_generator(object): return None solution = [] - for id, pkgs in enumerate(self.conflict_pkgs): - solution.append(pkgs[self.solution_ids[id]]) + for idx, pkgs in enumerate(self.conflict_pkgs): + solution.append(pkgs[self.solution_ids[idx]]) return solution - def _next(self, id=None): + def _next(self, id=None): # pylint: disable=redefined-builtin solution_ids = self.solution_ids conflict_pkgs = self.conflict_pkgs @@ -1112,34 +1108,24 @@ class _configuration_generator(object): if solution_ids[id] == len(conflict_pkgs[id])-1: if id > 0: return self._next(id=id-1) - else: - return False - else: - solution_ids[id] += 1 - for other_id in range(id+1, len(solution_ids)): - solution_ids[other_id] = 0 - return True + return False + + solution_ids[id] += 1 + for other_id in range(id+1, len(solution_ids)): + solution_ids[other_id] = 0 + return True -class _solution_candidate_generator(object): - class _value_helper(object): +class _solution_candidate_generator: + class _value_helper: def __init__(self, value=None): self.value = value def __eq__(self, other): - if isinstance(other, basestring): + if isinstance(other, str): return self.value == other - else: - return self.value == other.value + return self.value == other.value def __str__(self): return "%s" % (self.value,) - if sys.hexversion < 0x3000000: - - __unicode__ = __str__ - - def __str__(self): - return _unicode_encode(self.__unicode__(), - encoding=_encodings['content'], errors='backslashreplace') - def __init__(self, all_involved_flags): #A copy of all_involved_flags with all "cond" values #replaced by a _value_helper object. @@ -1171,7 +1157,7 @@ class _solution_candidate_generator(object): return self.all_involved_flags - def _next(self, id=None): + def _next(self, id=None): # pylint: disable=redefined-builtin values = self.conditional_values if not values: @@ -1183,12 +1169,11 @@ class _solution_candidate_generator(object): if values[id].value == "enabled": if id > 0: return self._next(id=id-1) - else: - return False - else: - values[id].value = "enabled" - for other_id in range(id+1, len(values)): - values[other_id].value = "disabled" - return True + return False + + values[id].value = "enabled" + for other_id in range(id+1, len(values)): + values[other_id].value = "disabled" + return True diff --git a/lib/_emerge/search.py b/lib/_emerge/search.py index be639dfa3..000c427e0 100644 --- a/lib/_emerge/search.py +++ b/lib/_emerge/search.py @@ -1,8 +1,6 @@ # Copyright 1999-2016 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - import difflib import re import portage @@ -17,7 +15,7 @@ from portage.util.iterators.MultiIterGroupBy import MultiIterGroupBy from _emerge.Package import Package -class search(object): +class search: # # class constants @@ -358,7 +356,7 @@ class search(object): def output(self): """Outputs the results of the search.""" - class msg(object): + class msg: @staticmethod def append(msg): writemsg_stdout(msg, noiselevel=-1) @@ -411,7 +409,7 @@ class search(object): desc = metadata["DESCRIPTION"] homepage = metadata["HOMEPAGE"] - license = metadata["LICENSE"] + license = metadata["LICENSE"] # pylint: disable=redefined-builtin if masked: msg.append(green("*") + " " + \ @@ -528,4 +526,3 @@ class search(object): else: result = "" return result - diff --git a/lib/_emerge/show_invalid_depstring_notice.py b/lib/_emerge/show_invalid_depstring_notice.py index e11ea65ed..a2f29eae6 100644 --- a/lib/_emerge/show_invalid_depstring_notice.py +++ b/lib/_emerge/show_invalid_depstring_notice.py @@ -32,4 +32,3 @@ def show_invalid_depstring_notice(parent_node, error_msg): msg2 = "".join("%s\n" % line for line in textwrap.wrap("".join(msg), 72)) writemsg_level(msg1 + msg2, level=logging.ERROR, noiselevel=-1) - diff --git a/lib/_emerge/stdout_spinner.py b/lib/_emerge/stdout_spinner.py index 670686adf..26592b7b0 100644 --- a/lib/_emerge/stdout_spinner.py +++ b/lib/_emerge/stdout_spinner.py @@ -7,7 +7,7 @@ import time from portage.output import darkgreen, green -class stdout_spinner(object): +class stdout_spinner: scroll_msgs = [ "Gentoo Rocks ("+platform.system()+")", "Thank you for using Gentoo. :)", diff --git a/lib/_emerge/unmerge.py b/lib/_emerge/unmerge.py index 8923e20ea..8de35a6ef 100644 --- a/lib/_emerge/unmerge.py +++ b/lib/_emerge/unmerge.py @@ -98,8 +98,8 @@ def _unmerge_display(root_config, myopts, unmerge_action, " can only be used with specific package names") print() return 1, {} - else: - global_unmerge = 1 + + global_unmerge = 1 localtree = vartree # process all arguments and add all @@ -605,4 +605,3 @@ def unmerge(root_config, myopts, unmerge_action, sets["selected"].unlock() return os.EX_OK - diff --git a/lib/portage/__init__.py b/lib/portage/__init__.py index ec667e478..2d8e4e8e4 100644 --- a/lib/portage/__init__.py +++ b/lib/portage/__init__.py @@ -1,8 +1,6 @@ -# Copyright 1998-2019 Gentoo Authors +# Copyright 1998-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - VERSION = "HEAD" # =========================================================================== @@ -136,10 +134,6 @@ except ImportError as e: sys.stderr.write(" "+str(e)+"\n\n") raise -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str - long = int # We use utf_8 encoding everywhere. Previously, we used # sys.getfilesystemencoding() for the 'merge' encoding, but that had @@ -170,45 +164,32 @@ _encodings = { 'stdio' : 'utf_8', } -if sys.hexversion >= 0x3000000: - def _decode_argv(argv): - # With Python 3, the surrogateescape encoding error handler makes it - # possible to access the original argv bytes, which can be useful - # if their actual encoding does no match the filesystem encoding. - fs_encoding = sys.getfilesystemencoding() - return [_unicode_decode(x.encode(fs_encoding, 'surrogateescape')) - for x in argv] +def _decode_argv(argv): + # With Python 3, the surrogateescape encoding error handler makes it + # possible to access the original argv bytes, which can be useful + # if their actual encoding does no match the filesystem encoding. + fs_encoding = sys.getfilesystemencoding() + return [_unicode_decode(x.encode(fs_encoding, 'surrogateescape')) + for x in argv] - def _unicode_encode(s, encoding=_encodings['content'], errors='backslashreplace'): - if isinstance(s, str): - s = s.encode(encoding, errors) - return s - def _unicode_decode(s, encoding=_encodings['content'], errors='replace'): - if isinstance(s, bytes): - s = str(s, encoding=encoding, errors=errors) - return s +def _unicode_encode(s, encoding=_encodings['content'], errors='backslashreplace'): + if isinstance(s, str): + s = s.encode(encoding, errors) + return s - _native_string = _unicode_decode -else: - def _decode_argv(argv): - return [_unicode_decode(x) for x in argv] +def _unicode_decode(s, encoding=_encodings['content'], errors='replace'): + if isinstance(s, bytes): + s = str(s, encoding=encoding, errors=errors) + return s - def _unicode_encode(s, encoding=_encodings['content'], errors='backslashreplace'): - if isinstance(s, unicode): - s = s.encode(encoding, errors) - return s - def _unicode_decode(s, encoding=_encodings['content'], errors='replace'): - if isinstance(s, bytes): - s = unicode(s, encoding=encoding, errors=errors) - return s +_native_string = _unicode_decode - _native_string = _unicode_encode -class _unicode_func_wrapper(object): +class _unicode_func_wrapper: """ Wraps a function, converts arguments from unicode to bytes, and return values to unicode from bytes. Function calls @@ -267,7 +248,7 @@ class _unicode_func_wrapper(object): return rval -class _unicode_module_wrapper(object): +class _unicode_module_wrapper: """ Wraps a module and wraps all functions with _unicode_func_wrapper. """ @@ -308,7 +289,7 @@ class _unicode_module_wrapper(object): cache[attr] = result return result -class _eintr_func_wrapper(object): +class _eintr_func_wrapper: """ Wraps a function and handles EINTR by calling the function as many times as necessary (until it returns without raising EINTR). @@ -417,7 +398,7 @@ bsd_chflags = None if platform.system() in ('FreeBSD',): # TODO: remove this class? - class bsd_chflags(object): + class bsd_chflags: chflags = os.chflags lchflags = os.lchflags @@ -479,7 +460,7 @@ def _eapi_is_deprecated(eapi): return eapi in _deprecated_eapis def eapi_is_supported(eapi): - if not isinstance(eapi, basestring): + if not isinstance(eapi, str): # Only call str() when necessary since with python2 it # can trigger UnicodeEncodeError if EAPI is corrupt. eapi = str(eapi) @@ -620,10 +601,10 @@ if VERSION == 'HEAD': head_timestamp = None if len(output_lines) > 3: try: - head_timestamp = long(output_lines[3]) + head_timestamp = int(output_lines[3]) except ValueError: pass - timestamp = long(time.time()) + timestamp = int(time.time()) if head_timestamp is not None and timestamp > head_timestamp: timestamp = timestamp - head_timestamp if not patchlevel: diff --git a/lib/portage/_emirrordist/Config.py b/lib/portage/_emirrordist/Config.py index c1f59f725..56732089e 100644 --- a/lib/portage/_emirrordist/Config.py +++ b/lib/portage/_emirrordist/Config.py @@ -1,20 +1,18 @@ -# Copyright 2013-2019 Gentoo Authors +# Copyright 2013-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import copy import io import logging import shelve -import sys import time import portage from portage import os from portage.package.ebuild.fetch import MirrorLayoutConfig from portage.util import grabdict, grablines -from portage.util._ShelveUnicodeWrapper import ShelveUnicodeWrapper -class Config(object): +class Config: def __init__(self, options, portdb, event_loop): self.options = options self.portdb = portdb @@ -97,7 +95,7 @@ class Config(object): return self._LogFormatter(line_format, log_func) - class _LogFormatter(object): + class _LogFormatter: __slots__ = ('_line_format', '_log_func') @@ -126,9 +124,6 @@ class Config(object): from bsddb3 import dbshelve db = dbshelve.open(db_file, flags=open_flag) - if sys.hexversion < 0x3000000: - db = ShelveUnicodeWrapper(db) - if self.options.dry_run: logging.warning("dry-run: %s db opened in readonly mode" % db_desc) if not isinstance(db, dict): diff --git a/lib/portage/_emirrordist/DeletionIterator.py b/lib/portage/_emirrordist/DeletionIterator.py index 3cbff2c3a..08985ed6c 100644 --- a/lib/portage/_emirrordist/DeletionIterator.py +++ b/lib/portage/_emirrordist/DeletionIterator.py @@ -7,7 +7,7 @@ import stat from portage import os from .DeletionTask import DeletionTask -class DeletionIterator(object): +class DeletionIterator: def __init__(self, config): self._config = config diff --git a/lib/portage/_emirrordist/FetchIterator.py b/lib/portage/_emirrordist/FetchIterator.py index 4ad797502..fe521c346 100644 --- a/lib/portage/_emirrordist/FetchIterator.py +++ b/lib/portage/_emirrordist/FetchIterator.py @@ -15,7 +15,7 @@ from .FetchTask import FetchTask from _emerge.CompositeTask import CompositeTask -class FetchIterator(object): +class FetchIterator: def __init__(self, config): self._config = config diff --git a/lib/portage/_emirrordist/FetchTask.py b/lib/portage/_emirrordist/FetchTask.py index 322de79ba..f2342362d 100644 --- a/lib/portage/_emirrordist/FetchTask.py +++ b/lib/portage/_emirrordist/FetchTask.py @@ -1,4 +1,4 @@ -# Copyright 2013-2019 Gentoo Authors +# Copyright 2013-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from __future__ import division @@ -7,9 +7,7 @@ import collections import errno import logging import random -import stat import subprocess -import sys import portage from portage import _encodings, _unicode_encode @@ -232,12 +230,11 @@ class FetchTask(CompositeTask): if self._fs_mirror_stack: self._fetch_fs(self._fs_mirror_stack.pop()) return - else: - uri = self._next_uri() - if uri is not None: - self._tried_uris.add(uri) - self._fetch_uri(uri) - return + uri = self._next_uri() + if uri is not None: + self._tried_uris.add(uri) + self._fetch_uri(uri) + return if self._tried_uris: msg = "all uris failed" @@ -352,14 +349,14 @@ class FetchTask(CompositeTask): self.returncode = os.EX_OK self.wait() return - else: - self._start_task( - FileCopier(src_path=src, dest_path=dest, - background=(self.background and - self._log_path is not None), - logfile=self._log_path), - self._fs_mirror_copier_exit) - return + + self._start_task( + FileCopier(src_path=src, dest_path=dest, + background=(self.background and + self._log_path is not None), + logfile=self._log_path), + self._fs_mirror_copier_exit) + return self._try_next_mirror() @@ -385,14 +382,9 @@ class FetchTask(CompositeTask): # Apply the timestamp from the source file, but # just rely on umask for permissions. try: - if sys.hexversion >= 0x3030000: - os.utime(copier.dest_path, - ns=(self._current_stat.st_mtime_ns, - self._current_stat.st_mtime_ns)) - else: - os.utime(copier.dest_path, - (self._current_stat[stat.ST_MTIME], - self._current_stat[stat.ST_MTIME])) + os.utime(copier.dest_path, + ns=(self._current_stat.st_mtime_ns, + self._current_stat.st_mtime_ns)) except OSError as e: msg = "%s %s utime failed unexpectedly: %s" % \ (self.distfile, current_mirror.name, e) @@ -605,11 +597,10 @@ class FetchTask(CompositeTask): def _select_hash(self): if default_hash_name in self.digests: return default_hash_name - else: - for hash_name in self.digests: - if hash_name != "size" and \ - hash_name in portage.checksum.get_valid_checksum_keys(): - return hash_name + for hash_name in self.digests: + if hash_name != "size" and \ + hash_name in portage.checksum.get_valid_checksum_keys(): + return hash_name return None diff --git a/lib/portage/_emirrordist/MirrorDistTask.py b/lib/portage/_emirrordist/MirrorDistTask.py index 8eb3081c6..e495ab9bf 100644 --- a/lib/portage/_emirrordist/MirrorDistTask.py +++ b/lib/portage/_emirrordist/MirrorDistTask.py @@ -1,9 +1,8 @@ -# Copyright 2013-2018 Gentoo Foundation +# Copyright 2013-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import errno import logging -import sys import time try: @@ -18,9 +17,6 @@ from _emerge.CompositeTask import CompositeTask from .FetchIterator import FetchIterator from .DeletionIterator import DeletionIterator -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - long = int class MirrorDistTask(CompositeTask): @@ -111,7 +107,7 @@ class MirrorDistTask(CompositeTask): recycle_db[filename] = (st.st_size, start_time) else: r_size, r_time = value - if long(r_size) != st.st_size: + if int(r_size) != st.st_size: recycle_db[filename] = (st.st_size, start_time) elif r_time + r_deletion_delay < start_time: if self._config.options.dry_run: diff --git a/lib/portage/_emirrordist/main.py b/lib/portage/_emirrordist/main.py index ce0c2929f..23f6468c5 100644 --- a/lib/portage/_emirrordist/main.py +++ b/lib/portage/_emirrordist/main.py @@ -1,4 +1,4 @@ -# Copyright 2013-2015 Gentoo Foundation +# Copyright 2013-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import argparse @@ -14,9 +14,6 @@ from portage.util._eventloop.global_event_loop import global_event_loop from .Config import Config from .MirrorDistTask import MirrorDistTask -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - long = int seconds_per_day = 24 * 60 * 60 @@ -318,7 +315,7 @@ def emirrordist_main(args): parser.error("--scheduled-deletion-log requires --deletion-db") if options.deletion_delay is not None: - options.deletion_delay = long(options.deletion_delay) + options.deletion_delay = int(options.deletion_delay) if options.deletion_db is None: parser.error("--deletion-delay requires --deletion-db") @@ -391,7 +388,7 @@ def emirrordist_main(args): if options.recycle_deletion_delay is not None: options.recycle_deletion_delay = \ - long(options.recycle_deletion_delay) + int(options.recycle_deletion_delay) if options.fetch_log_dir is not None: options.fetch_log_dir = normalize_path( diff --git a/lib/portage/_global_updates.py b/lib/portage/_global_updates.py index 6a595da56..730ade592 100644 --- a/lib/portage/_global_updates.py +++ b/lib/portage/_global_updates.py @@ -149,8 +149,7 @@ def _do_global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): if portdb.match(atoma): world_warnings.add((atoma, atomb)) return True - else: - return False + return False for update_cmd in myupd: for pos, atom in enumerate(world_list): diff --git a/lib/portage/_legacy_globals.py b/lib/portage/_legacy_globals.py index 45113d150..a9f8aa62d 100644 --- a/lib/portage/_legacy_globals.py +++ b/lib/portage/_legacy_globals.py @@ -15,7 +15,7 @@ def _get_legacy_global(name): constructed.add(name) return getattr(portage, name) - elif name in ('mtimedb', 'mtimedbfile'): + if name in ('mtimedb', 'mtimedbfile'): portage.mtimedbfile = os.path.join(portage.settings['EROOT'], CACHE_PATH, "mtimedb") constructed.add('mtimedbfile') diff --git a/lib/portage/_selinux.py b/lib/portage/_selinux.py index 49e2e8e58..aba40c2dd 100644 --- a/lib/portage/_selinux.py +++ b/lib/portage/_selinux.py @@ -5,7 +5,6 @@ # the whole _selinux module itself will be wrapped. import os import shutil -import sys import warnings try: @@ -23,8 +22,6 @@ def copyfile(src, dest): dest = _native_string(dest, encoding=_encodings['fs'], errors='strict') (rc, ctx) = selinux.lgetfilecon(src) if rc < 0: - if sys.hexversion < 0x3000000: - src = _unicode_decode(src, encoding=_encodings['fs'], errors='replace') raise OSError(_("copyfile: Failed getting context of \"%s\".") % src) setfscreate(ctx) @@ -48,8 +45,6 @@ def mkdir(target, refdir): refdir = _native_string(refdir, encoding=_encodings['fs'], errors='strict') (rc, ctx) = selinux.getfilecon(refdir) if rc < 0: - if sys.hexversion < 0x3000000: - refdir = _unicode_decode(refdir, encoding=_encodings['fs'], errors='replace') raise OSError( _("mkdir: Failed getting context of reference directory \"%s\".") \ % refdir) @@ -65,8 +60,6 @@ def rename(src, dest): dest = _native_string(dest, encoding=_encodings['fs'], errors='strict') (rc, ctx) = selinux.lgetfilecon(src) if rc < 0: - if sys.hexversion < 0x3000000: - src = _unicode_decode(src, encoding=_encodings['fs'], errors='replace') raise OSError(_("rename: Failed getting context of \"%s\".") % src) setfscreate(ctx) @@ -98,8 +91,6 @@ def setexec(ctx="\n"): portage.writemsg("!!! %s\n" % msg, noiselevel=-1) if rc < 0: - if sys.hexversion < 0x3000000: - ctx = _unicode_decode(ctx, encoding=_encodings['content'], errors='replace') if selinux.security_getenforce() == 1: raise OSError(_("Failed setting exec() context \"%s\".") % ctx) else: @@ -110,12 +101,10 @@ def setexec(ctx="\n"): def setfscreate(ctx="\n"): ctx = _native_string(ctx, encoding=_encodings['content'], errors='strict') if selinux.setfscreatecon(ctx) < 0: - if sys.hexversion < 0x3000000: - ctx = _unicode_decode(ctx, encoding=_encodings['content'], errors='replace') raise OSError( _("setfscreate: Failed setting fs create context \"%s\".") % ctx) -class spawn_wrapper(object): +class spawn_wrapper: """ Create a wrapper function for the given spawn function. When the wrapper is called, it will adjust the arguments such that setexec() to be called @@ -148,8 +137,6 @@ def symlink(target, link, reflnk): reflnk = _native_string(reflnk, encoding=_encodings['fs'], errors='strict') (rc, ctx) = selinux.lgetfilecon(reflnk) if rc < 0: - if sys.hexversion < 0x3000000: - reflnk = _unicode_decode(reflnk, encoding=_encodings['fs'], errors='replace') raise OSError( _("symlink: Failed getting context of reference symlink \"%s\".") \ % reflnk) diff --git a/lib/portage/_sets/__init__.py b/lib/portage/_sets/__init__.py index a569b273b..6c6df4cca 100644 --- a/lib/portage/_sets/__init__.py +++ b/lib/portage/_sets/__init__.py @@ -29,17 +29,16 @@ SETPREFIX = "@" def get_boolean(options, name, default): if not name in options: return default - elif options[name].lower() in ("1", "yes", "on", "true"): + if options[name].lower() in ("1", "yes", "on", "true"): return True - elif options[name].lower() in ("0", "no", "off", "false"): + if options[name].lower() in ("0", "no", "off", "false"): return False - else: - raise SetConfigError(_("invalid value '%(value)s' for option '%(option)s'") % {"value": options[name], "option": name}) + raise SetConfigError(_("invalid value '%(value)s' for option '%(option)s'") % {"value": options[name], "option": name}) class SetConfigError(Exception): pass -class SetConfig(object): +class SetConfig: def __init__(self, paths, settings, trees): self._parser = SafeConfigParser( defaults={ diff --git a/lib/portage/_sets/base.py b/lib/portage/_sets/base.py index aba295602..06e0a3cd4 100644 --- a/lib/portage/_sets/base.py +++ b/lib/portage/_sets/base.py @@ -1,18 +1,14 @@ -# Copyright 2007-2018 Gentoo Foundation +# Copyright 2007-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -import sys from portage.dep import Atom, ExtendedAtomDict, best_match_to_list, match_from_list from portage.exception import InvalidAtom from portage.versions import cpv_getkey -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str OPERATIONS = ["merge", "unmerge"] -class PackageSet(object): +class PackageSet: # Set this to operations that are supported by your subclass. While # technically there is no difference between "merge" and "unmerge" regarding # package sets, the latter doesn't make sense for some sets like "system" @@ -46,9 +42,6 @@ class PackageSet(object): self._load() return bool(self._atoms or self._nonatoms) - if sys.hexversion < 0x3000000: - __nonzero__ = __bool__ - def supportsOperation(self, op): if not op in OPERATIONS: raise ValueError(op) @@ -74,7 +67,7 @@ class PackageSet(object): self._nonatoms.clear() for a in atoms: if not isinstance(a, Atom): - if isinstance(a, basestring): + if isinstance(a, str): a = a.strip() if not a: continue @@ -107,8 +100,7 @@ class PackageSet(object): def getMetadata(self, key): if hasattr(self, key.lower()): return getattr(self, key.lower()) - else: - return "" + return "" def _updateAtomMap(self, atoms=None): """Update self._atommap for specific atoms or all atoms.""" diff --git a/lib/portage/_sets/dbapi.py b/lib/portage/_sets/dbapi.py index 5c600ec9e..a9e1b6880 100644 --- a/lib/portage/_sets/dbapi.py +++ b/lib/portage/_sets/dbapi.py @@ -358,8 +358,7 @@ class AgeSet(EverythingSet): if ((self._mode == "older" and age <= self._age) \ or (self._mode == "newer" and age >= self._age)): return False - else: - return True + return True def singleBuilder(cls, options, settings, trees): mode = options.get("mode", "older") @@ -394,8 +393,7 @@ class DateSet(EverythingSet): if ((self._mode == "older" and date < self._date) \ or (self._mode == "newer" and date > self._date)): return True - else: - return False + return False def singleBuilder(cls, options, settings, trees): vardbapi = trees["vartree"].dbapi @@ -418,9 +416,9 @@ class DateSet(EverythingSet): elif len(formats) > 1: raise SetConfigError(_("no more than one of these options is allowed: 'package', 'filestamp', 'seconds', 'date'")) - format = formats[0] + setformat = formats[0] - if (format == "package"): + if (setformat == "package"): package = options.get("package") try: cpv = vardbapi.match(package)[0] @@ -428,13 +426,13 @@ class DateSet(EverythingSet): date = int(date) except (KeyError, ValueError): raise SetConfigError(_("cannot determine installation date of package %s") % package) - elif (format == "filestamp"): + elif (setformat == "filestamp"): filestamp = options.get("filestamp") try: date = int(os.stat(filestamp).st_mtime) except (OSError, ValueError): raise SetConfigError(_("cannot determine 'filestamp' of '%s'") % filestamp) - elif (format == "seconds"): + elif (setformat == "seconds"): try: date = int(options.get("seconds")) except ValueError: @@ -508,25 +506,24 @@ class ChangedDepsSet(PackageSet): if isinstance(depatom, list): # process the nested list. return [clean_subslots(x, usel) for x in depatom] - else: - try: - # this can be either an atom or some special operator. - # in the latter case, we get InvalidAtom and pass it as-is. - a = Atom(depatom) - except InvalidAtom: - return depatom - else: - # if we're processing portdb, we need to evaluate USE flag - # dependency conditionals to make them match vdb. this - # requires passing the list of USE flags, so we reuse it - # as conditional for the operation as well. - if usel is not None: - a = a.evaluate_conditionals(usel) - - # replace slot operator := dependencies with plain := - # since we can't properly compare expanded slots - # in vardb to abstract slots in portdb. - return subslot_repl_re.sub(':=', a) + + try: + # this can be either an atom or some special operator. + # in the latter case, we get InvalidAtom and pass it as-is. + a = Atom(depatom) + except InvalidAtom: + return depatom + # if we're processing portdb, we need to evaluate USE flag + # dependency conditionals to make them match vdb. this + # requires passing the list of USE flags, so we reuse it + # as conditional for the operation as well. + if usel is not None: + a = a.evaluate_conditionals(usel) + + # replace slot operator := dependencies with plain := + # since we can't properly compare expanded slots + # in vardb to abstract slots in portdb. + return subslot_repl_re.sub(':=', a) # get all *DEPEND variables from vdb & portdb and compare them. # we need to do some cleaning up & expansion to make matching diff --git a/lib/portage/cache/__init__.py b/lib/portage/cache/__init__.py index e7fe599f0..e0b8b827f 100644 --- a/lib/portage/cache/__init__.py +++ b/lib/portage/cache/__init__.py @@ -1,4 +1,3 @@ # Copyright: 2005 Gentoo Foundation # Author(s): Brian Harring (ferringb@gentoo.org) # License: GPL2 - diff --git a/lib/portage/cache/anydbm.py b/lib/portage/cache/anydbm.py index 88d85b0da..4cdf264e2 100644 --- a/lib/portage/cache/anydbm.py +++ b/lib/portage/cache/anydbm.py @@ -4,32 +4,16 @@ from __future__ import absolute_import -try: - import anydbm as anydbm_module -except ImportError: - # python 3.x - import dbm as anydbm_module +import dbm try: import dbm.gnu as gdbm except ImportError: - try: - import gdbm - except ImportError: - gdbm = None + gdbm = None -try: - from dbm import whichdb -except ImportError: - from whichdb import whichdb - -try: - import cPickle as pickle -except ImportError: - import pickle +import pickle from portage import _unicode_encode from portage import os -import sys from portage.cache import fs_template from portage.cache import cache_errors @@ -53,15 +37,15 @@ class database(fs_template.FsBased): self._db_path = os.path.join(self.location, fs_template.gen_label(self.location, self.label)+default_db) self.__db = None mode = "w" - if whichdb(self._db_path) in ("dbm.gnu", "gdbm"): + if dbm.whichdb(self._db_path) in ("dbm.gnu", "gdbm"): # Allow multiple concurrent writers (see bug #53607). mode += "u" try: # dbm.open() will not work with bytes in python-3.1: # TypeError: can't concat bytes to str - self.__db = anydbm_module.open(self._db_path, + self.__db = dbm.open(self._db_path, mode, self._perms) - except anydbm_module.error: + except dbm.error: # XXX handle this at some point try: self._ensure_dirs() @@ -75,14 +59,14 @@ class database(fs_template.FsBased): # dbm.open() will not work with bytes in python-3.1: # TypeError: can't concat bytes to str if gdbm is None: - self.__db = anydbm_module.open(self._db_path, + self.__db = dbm.open(self._db_path, "c", self._perms) else: # Prefer gdbm type if available, since it allows # multiple concurrent writers (see bug #53607). self.__db = gdbm.open(self._db_path, "cu", self._perms) - except anydbm_module.error as e: + except dbm.error as e: raise cache_errors.InitializationError(self.__class__, e) self._ensure_access(self._db_path) @@ -112,5 +96,5 @@ class database(fs_template.FsBased): self.__db.sync() self.__db.close() - if sys.hexversion >= 0x3000000: - items = iteritems + # TODO: do we need iteritems()? + items = iteritems diff --git a/lib/portage/cache/ebuild_xattr.py b/lib/portage/cache/ebuild_xattr.py index 33a40fdba..6b42d79df 100644 --- a/lib/portage/cache/ebuild_xattr.py +++ b/lib/portage/cache/ebuild_xattr.py @@ -38,9 +38,9 @@ class database(fs_template.FsBased): try: return int(self.__get(path,'value_max_len')) except NoValueException as e: - max = self.__calc_max(path) - self.__set(path,'value_max_len',str(max)) - return max + maxattrlength = self.__calc_max(path) + self.__set(path, 'value_max_len', str(maxattrlength)) + return maxattrlength def __calc_max(self,path): """ Find out max attribute length supported by the file system """ @@ -90,8 +90,7 @@ class database(fs_template.FsBased): except IOError as e: if not default is None and errno.ENODATA == e.errno: return default - else: - raise NoValueException() + raise NoValueException() def __remove(self,path,key): xattr.remove(path,key,namespace=self.ns) @@ -102,48 +101,48 @@ class database(fs_template.FsBased): def _getitem(self, cpv): values = {} path = self.__get_path(cpv) - all = {} - for tuple in xattr.get_all(path,namespace=self.ns): - key,value = tuple - all[key] = value + attrs = { + key: value + for key, value in xattr.get_all(path, namespace=self.ns) + } if not '_mtime_' in all: raise KeyError(cpv) # We default to '' like other caches for key in self.keys: - attr_value = all.get(key,'1:') + attr_value = attrs.get(key,'1:') parts,sep,value = attr_value.partition(':') parts = int(parts) if parts > 1: for i in range(1,parts): - value += all.get(key+str(i)) + value += attrs.get(key+str(i)) values[key] = value return values def _setitem(self, cpv, values): path = self.__get_path(cpv) - max = self.max_len + max_len = self.max_len for key,value in values.items(): # mtime comes in as long so need to convert to strings s = str(value) # We need to split long values value_len = len(s) parts = 0 - if value_len > max: + if value_len > max_len: # Find out how many parts we need - parts = value_len/max - if value_len % max > 0: + parts = value_len/max_len + if value_len % max_len > 0: parts += 1 # Only the first entry carries the number of parts - self.__set(path,key,'%s:%s'%(parts,s[0:max])) + self.__set(path,key,'%s:%s'%(parts,s[0:max_len])) # Write out the rest for i in range(1,parts): - start = i * max - val = s[start:start+max] + start = i * max_len + val = s[start:start+max_len] self.__set(path,key+str(i),val) else: self.__set(path,key,"%s:%s"%(1,s)) diff --git a/lib/portage/cache/flat_hash.py b/lib/portage/cache/flat_hash.py index 451ea9e51..7d5b44511 100644 --- a/lib/portage/cache/flat_hash.py +++ b/lib/portage/cache/flat_hash.py @@ -1,15 +1,12 @@ -# Copyright 2005-2019 Gentoo Authors +# Copyright 2005-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 # Author(s): Brian Harring (ferringb@gentoo.org) -from __future__ import unicode_literals - from portage.cache import fs_template from portage.cache import cache_errors import errno import io import stat -import sys import tempfile import os as _os from portage import os @@ -18,9 +15,6 @@ from portage import _unicode_encode from portage.exception import InvalidData from portage.versions import _pkg_str -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - long = int class database(fs_template.FsBased): diff --git a/lib/portage/cache/fs_template.py b/lib/portage/cache/fs_template.py index e3c3c12c2..64050f439 100644 --- a/lib/portage/cache/fs_template.py +++ b/lib/portage/cache/fs_template.py @@ -1,9 +1,8 @@ -# Copyright 2005-2014 Gentoo Foundation +# Copyright 2005-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 # Author(s): Brian Harring (ferringb@gentoo.org) import os as _os -import sys from portage.cache import template from portage import os @@ -14,9 +13,6 @@ lazyimport(globals(), ) del lazyimport -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - long = int class FsBased(template.database): """template wrapping fs needed options, and providing _ensure_access as a way to @@ -44,7 +40,7 @@ class FsBased(template.database): try: apply_permissions(path, gid=self._gid, mode=self._perms) if mtime != -1: - mtime=long(mtime) + mtime=int(mtime) os.utime(path, (mtime, mtime)) except (PortageException, EnvironmentError): return False @@ -59,8 +55,8 @@ class FsBased(template.database): path = self.location base='/' - for dir in path.lstrip(os.path.sep).rstrip(os.path.sep).split(os.path.sep): - base = os.path.join(base,dir) + for d in path.lstrip(os.path.sep).rstrip(os.path.sep).split(os.path.sep): + base = os.path.join(base,d) if ensure_dirs(base): # We only call apply_permissions if ensure_dirs created # a new directory, so as not to interfere with @@ -90,4 +86,3 @@ def gen_label(base, label): label = os.path.join(*(label.rstrip(os.path.sep).split(os.path.sep))) tail = os.path.split(label)[1] return "%s-%X" % (tail, abs(label.__hash__())) - diff --git a/lib/portage/cache/index/IndexStreamIterator.py b/lib/portage/cache/index/IndexStreamIterator.py index 972aee116..4ecfad02b 100644 --- a/lib/portage/cache/index/IndexStreamIterator.py +++ b/lib/portage/cache/index/IndexStreamIterator.py @@ -1,7 +1,7 @@ # Copyright 2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -class IndexStreamIterator(object): +class IndexStreamIterator: def __init__(self, f, parser): diff --git a/lib/portage/cache/index/pkg_desc_index.py b/lib/portage/cache/index/pkg_desc_index.py index dbcbb8313..c3bcbb933 100644 --- a/lib/portage/cache/index/pkg_desc_index.py +++ b/lib/portage/cache/index/pkg_desc_index.py @@ -1,22 +1,15 @@ -# Copyright 2014 Gentoo Foundation +# Copyright 2014-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - import collections -import sys from portage.versions import _pkg_str -if sys.hexversion >= 0x3000000: - _unicode = str -else: - _unicode = unicode pkg_desc_index_node = collections.namedtuple("pkg_desc_index_node", ["cp", "cpv_list", "desc"]) -class pkg_node(_unicode): +class pkg_node(str): """ A minimal package node class. For performance reasons, inputs are not validated. @@ -29,7 +22,7 @@ class pkg_node(_unicode): self.__dict__['build_time'] = None def __new__(cls, cp, version, repo=None): - return _unicode.__new__(cls, cp + "-" + version) + return str.__new__(cls, cp + "-" + version) def __setattr__(self, name, value): raise AttributeError("pkg_node instances are immutable", diff --git a/lib/portage/cache/mappings.py b/lib/portage/cache/mappings.py index 0432fdf60..5933981b9 100644 --- a/lib/portage/cache/mappings.py +++ b/lib/portage/cache/mappings.py @@ -5,10 +5,9 @@ __all__ = ["Mapping", "MutableMapping", "UserDict", "ProtectedDict", "LazyLoad", "slot_dict_class"] -import sys import weakref -class Mapping(object): +class Mapping: """ In python-3.0, the UserDict.DictMixin class has been replaced by Mapping and MutableMapping from the collections module, but 2to3 @@ -25,9 +24,6 @@ class Mapping(object): def __iter__(self): return iter(self.keys()) - def keys(self): - return list(self.__iter__()) - def __contains__(self, key): try: value = self[key] @@ -46,12 +42,6 @@ class Mapping(object): for _, v in self.items(): yield v - def values(self): - return [v for _, v in self.iteritems()] - - def items(self): - return list(self.iteritems()) - def get(self, key, default=None): try: return self[key] @@ -64,10 +54,10 @@ class Mapping(object): def __len__(self): return len(list(self)) - if sys.hexversion >= 0x3000000: - items = iteritems - keys = __iter__ - values = itervalues + # TODO: do we need to keep iter*? + items = iteritems + keys = __iter__ + values = itervalues class MutableMapping(Mapping): """ @@ -184,8 +174,8 @@ class UserDict(MutableMapping): def clear(self): self.data.clear() - if sys.hexversion >= 0x3000000: - keys = __iter__ + keys = __iter__ + class ProtectedDict(MutableMapping): """ @@ -234,8 +224,8 @@ class ProtectedDict(MutableMapping): def __contains__(self, key): return key in self.new or (key not in self.blacklist and key in self.orig) - if sys.hexversion >= 0x3000000: - keys = __iter__ + keys = __iter__ + class LazyLoad(Mapping): """ @@ -252,7 +242,7 @@ class LazyLoad(Mapping): def __getitem__(self, key): if key in self.d: return self.d[key] - elif self.pull != None: + if self.pull != None: self.d.update(self.pull()) self.pull = None return self.d[key] @@ -266,13 +256,13 @@ class LazyLoad(Mapping): def __contains__(self, key): if key in self.d: return True - elif self.pull != None: + if self.pull != None: self.d.update(self.pull()) self.pull = None return key in self.d - if sys.hexversion >= 0x3000000: - keys = __iter__ + keys = __iter__ + _slot_dict_classes = weakref.WeakValueDictionary() @@ -298,7 +288,7 @@ def slot_dict_class(keys, prefix="_val_"): v = _slot_dict_classes.get((keys_set, prefix)) if v is None: - class SlotDict(object): + class SlotDict: allowed_keys = keys_set _prefix = prefix @@ -328,9 +318,6 @@ def slot_dict_class(keys, prefix="_val_"): l += 1 return l - def keys(self): - return list(self) - def iteritems(self): prefix = self._prefix for k in self.allowed_keys: @@ -339,16 +326,10 @@ def slot_dict_class(keys, prefix="_val_"): except AttributeError: pass - def items(self): - return list(self.iteritems()) - def itervalues(self): for k, v in self.iteritems(): yield v - def values(self): - return list(self.itervalues()) - def __delitem__(self, k): try: delattr(self, self._prefix + k) @@ -447,10 +428,9 @@ def slot_dict_class(keys, prefix="_val_"): def __repr__(self): return repr(dict(self.iteritems())) - if sys.hexversion >= 0x3000000: - items = iteritems - keys = __iter__ - values = itervalues + items = iteritems + keys = __iter__ + values = itervalues v = SlotDict _slot_dict_classes[v.allowed_keys] = v diff --git a/lib/portage/cache/metadata.py b/lib/portage/cache/metadata.py index 45a057d08..db81b8ba1 100644 --- a/lib/portage/cache/metadata.py +++ b/lib/portage/cache/metadata.py @@ -1,11 +1,10 @@ -# Copyright 2005-2018 Gentoo Foundation +# Copyright 2005-2020 Gentoo Authors # Author(s): Brian Harring (ferringb@gentoo.org) # License: GPL2 import errno import re import stat -import sys from operator import attrgetter from portage import os from portage import _encodings @@ -15,10 +14,6 @@ import portage.eclass_cache from portage.cache.template import reconstruct_eclasses from portage.cache.mappings import ProtectedDict -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str - long = int # this is the old cache format, flat_list. count maintained here. magic_line_count = 22 @@ -76,7 +71,7 @@ class database(flat_hash.database): raise cache_errors.CacheCorruption(cpv, e) else: d["_eclasses_"] = {} - elif isinstance(d["_eclasses_"], basestring): + elif isinstance(d["_eclasses_"], str): # We skip this if flat_hash.database._parse_data() was called above # because it calls reconstruct_eclasses() internally. d["_eclasses_"] = reconstruct_eclasses(None, d["_eclasses_"]) diff --git a/lib/portage/cache/sql_template.py b/lib/portage/cache/sql_template.py index d023b1b5d..b87612f2d 100644 --- a/lib/portage/cache/sql_template.py +++ b/lib/portage/cache/sql_template.py @@ -2,7 +2,6 @@ # Author(s): Brian Harring (ferringb@gentoo.org) # License: GPL2 -import sys from portage.cache import template, cache_errors from portage.cache.template import reconstruct_eclasses @@ -296,6 +295,5 @@ class SQLDatabase(template.database): return [ row[0] for row in self.con.fetchall() ] - if sys.hexversion >= 0x3000000: - items = iteritems - keys = __iter__ + items = iteritems + keys = __iter__ diff --git a/lib/portage/cache/sqlite.py b/lib/portage/cache/sqlite.py index 69150f679..02bf3fcde 100644 --- a/lib/portage/cache/sqlite.py +++ b/lib/portage/cache/sqlite.py @@ -1,10 +1,9 @@ -# Copyright 1999-2014 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import division, unicode_literals +from __future__ import division import re -import sys from portage.cache import fs_template from portage.cache import cache_errors from portage import os @@ -12,9 +11,6 @@ from portage import _unicode_decode from portage.util import writemsg from portage.localization import _ -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str class database(fs_template.FsBased): @@ -64,7 +60,7 @@ class database(fs_template.FsBased): def _db_escape_string(self, s): """meta escaping, returns quoted string for use in sql statements""" - if not isinstance(s, basestring): + if not isinstance(s, str): # Avoid potential UnicodeEncodeError in python-2.x by # only calling str() when it's absolutely necessary. s = str(s) @@ -267,10 +263,9 @@ class database(fs_template.FsBased): result = cursor.fetchall() if len(result) == 0: return False - elif len(result) == 1: + if len(result) == 1: return True - else: - raise cache_errors.CacheCorruption(cpv, "key is not unique") + raise cache_errors.CacheCorruption(cpv, "key is not unique") def __iter__(self): """generator for walking the dir struct""" diff --git a/lib/portage/cache/template.py b/lib/portage/cache/template.py index 6b4878347..509f8a2dc 100644 --- a/lib/portage/cache/template.py +++ b/lib/portage/cache/template.py @@ -1,23 +1,15 @@ -# Copyright 2005-2014 Gentoo Foundation +# Copyright 2005-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 # Author(s): Brian Harring (ferringb@gentoo.org) from portage.cache import cache_errors from portage.cache.cache_errors import InvalidRestriction from portage.cache.mappings import ProtectedDict -import sys import warnings import operator -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - _unicode = str - basestring = str - long = int -else: - _unicode = unicode -class database(object): +class database: # this is for metadata/cache transfer. # basically flags the cache needs be updated when transfered cache to cache. # leave this. @@ -94,10 +86,10 @@ class database(object): d.pop('_mtime_', None) else: try: - mtime = long(mtime) + mtime = int(mtime) except ValueError: raise cache_errors.CacheCorruption(cpv, - '_mtime_ conversion to long failed: %s' % (mtime,)) + '_mtime_ conversion to int failed: %s' % (mtime,)) d['_mtime_'] = mtime return d @@ -178,9 +170,6 @@ class database(object): def has_key(self, cpv): return cpv in self - def keys(self): - return list(self) - def iterkeys(self): return iter(self) @@ -188,9 +177,6 @@ class database(object): for x in self: yield (x, self[x]) - def items(self): - return list(self.iteritems()) - def sync(self, rate=0): self.sync_rate = rate if(rate == 0): @@ -278,7 +264,7 @@ class database(object): for key,match in match_dict.items(): # XXX this sucks. try: - if isinstance(match, basestring): + if isinstance(match, str): restricts[key] = re.compile(match).match else: restricts[key] = re.compile(match[0],match[1]).match @@ -297,16 +283,16 @@ class database(object): if cont: yield cpv - if sys.hexversion >= 0x3000000: - keys = __iter__ - items = iteritems + keys = __iter__ + items = iteritems + _keysorter = operator.itemgetter(0) def serialize_eclasses(eclass_dict, chf_type='mtime', paths=True): """takes a dict, returns a string representing said dict""" """The "new format", which causes older versions of <portage-2.1.2 to - traceback with a ValueError due to failed long() conversion. This format + traceback with a ValueError due to failed int() conversion. This format isn't currently written, but the the capability to read it is already built in. return "\t".join(["%s\t%s" % (k, str(v)) \ @@ -335,7 +321,7 @@ def _md5_deserializer(md5): _chf_deserializers = { 'md5': _md5_deserializer, - 'mtime': long, + 'mtime': int, } diff --git a/lib/portage/checksum.py b/lib/portage/checksum.py index 4174638e6..8f01f6ac4 100644 --- a/lib/portage/checksum.py +++ b/lib/portage/checksum.py @@ -12,7 +12,6 @@ import errno import functools import hashlib import stat -import sys import subprocess import tempfile @@ -52,7 +51,7 @@ def _open_file(filename): else: raise -class _generate_hash_function(object): +class _generate_hash_function: __slots__ = ("_hashobject",) @@ -159,7 +158,7 @@ if False: import binascii import pygcrypt.hashcontext - class GCryptHashWrapper(object): + class GCryptHashWrapper: def __init__(self, algo): self._obj = pygcrypt.hashcontext.HashContext(algo=algo, secure=False) @@ -287,7 +286,7 @@ if "WHIRLPOOL" not in hashfunc_map: # There is only one implementation for size -class SizeHash(object): +class SizeHash: def checksum_file(self, filename): size = os.stat(filename).st_size return (size, size) @@ -361,7 +360,7 @@ def _filter_unaccelarated_hashes(digests): return digests -class _hash_filter(object): +class _hash_filter: """ Implements filtering for PORTAGE_CHECKSUM_FILTER. """ @@ -383,7 +382,7 @@ class _hash_filter(object): for token in self._tokens: if token in matches: return True - elif token[:1] == "-": + if token[:1] == "-": if token[1:] in matches: return False return False diff --git a/lib/portage/const.py b/lib/portage/const.py index 146808fea..50412c058 100644 --- a/lib/portage/const.py +++ b/lib/portage/const.py @@ -2,8 +2,6 @@ # Copyright 1998-2019 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - # =========================================================================== # autotool supplied constants. # =========================================================================== diff --git a/lib/portage/cvstree.py b/lib/portage/cvstree.py index 87bbed8bb..796462af2 100644 --- a/lib/portage/cvstree.py +++ b/lib/portage/cvstree.py @@ -7,16 +7,12 @@ from __future__ import print_function import io import re import stat -import sys import time from portage import os from portage import _encodings from portage import _unicode_encode -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - long = int # [D]/Name/Version/Date/Flags/Tags @@ -33,10 +29,9 @@ def pathdata(entries, path): return None if mytarget in myentries["dirs"]: return myentries["dirs"][mytarget] - elif mytarget in myentries["files"]: + if mytarget in myentries["files"]: return myentries["files"][mytarget] - else: - return None + return None def fileat(entries, path): return pathdata(entries, path) @@ -202,14 +197,14 @@ def findall(entries, recursive=0, basedir=""): return [mynew, mychanged, mymissing, myunadded, myremoved] ignore_list = re.compile(r"(^|/)(RCS(|LOG)|SCCS|CVS(|\.adm)|cvslog\..*|tags|TAGS|\.(make\.state|nse_depinfo)|.*~|(\.|)#.*|,.*|_$.*|.*\$|\.del-.*|.*\.(old|BAK|bak|orig|rej|a|olb|o|obj|so|exe|Z|elc|ln)|core)$") -def apply_cvsignore_filter(list): +def apply_cvsignore_filter(files): x = 0 - while x < len(list): - if ignore_list.match(list[x].split("/")[-1]): - list.pop(x) + while x < len(files): + if ignore_list.match(files[x].split("/")[-1]): + files.pop(x) else: x += 1 - return list + return files def getentries(mydir, recursive=0): """Scans the given directory and returns a datadict of all the entries in diff --git a/lib/portage/data.py b/lib/portage/data.py index 20a8d1ba7..d2d356f95 100644 --- a/lib/portage/data.py +++ b/lib/portage/data.py @@ -2,7 +2,10 @@ # Copyright 1998-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -import os, pwd, grp, platform, sys +import grp +import os +import platform +import pwd from portage.const import PORTAGE_GROUPNAME, PORTAGE_USERNAME, EPREFIX import portage @@ -198,10 +201,9 @@ def _get_global(k): if k == 'portage_gid': return portage_gid - elif k == 'portage_uid': + if k == 'portage_uid': return portage_uid - else: - raise AssertionError('unknown name: %s' % k) + raise AssertionError('unknown name: %s' % k) elif k == 'userpriv_groups': v = [_get_global('portage_gid')] diff --git a/lib/portage/dbapi/DummyTree.py b/lib/portage/dbapi/DummyTree.py index 6579e88e2..5ad7389d4 100644 --- a/lib/portage/dbapi/DummyTree.py +++ b/lib/portage/dbapi/DummyTree.py @@ -1,7 +1,7 @@ # Copyright 2015 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -class DummyTree(object): +class DummyTree: """ Most internal code only accesses the "dbapi" attribute of the binarytree, portagetree, and vartree classes. DummyTree is useful diff --git a/lib/portage/dbapi/IndexedPortdb.py b/lib/portage/dbapi/IndexedPortdb.py index 510e0278c..5f1cb5bd1 100644 --- a/lib/portage/dbapi/IndexedPortdb.py +++ b/lib/portage/dbapi/IndexedPortdb.py @@ -17,7 +17,7 @@ from portage.cache.index.pkg_desc_index import \ from portage.util.iterators.MultiIterGroupBy import MultiIterGroupBy from portage.versions import _pkg_str -class IndexedPortdb(object): +class IndexedPortdb: """ A portdbapi interface that uses a package description index to improve performance. If the description index is missing for a @@ -90,7 +90,7 @@ class IndexedPortdb(object): if index_missing: self._unindexed_cp_map = {} - class _NonIndexedStream(object): + class _NonIndexedStream: def __iter__(self_): for cp in self._portdb.cp_all( trees = index_missing): @@ -159,8 +159,7 @@ class IndexedPortdb(object): if atom == atom.cp: return cp_list[:] - else: - return portage.match_from_list(atom, cp_list) + return portage.match_from_list(atom, cp_list) def aux_get(self, cpv, attrs, myrepo=None): if len(attrs) == 1 and attrs[0] == "DESCRIPTION": diff --git a/lib/portage/dbapi/IndexedVardb.py b/lib/portage/dbapi/IndexedVardb.py index e2910b27f..a04e3074c 100644 --- a/lib/portage/dbapi/IndexedVardb.py +++ b/lib/portage/dbapi/IndexedVardb.py @@ -6,7 +6,7 @@ from portage.dep import Atom from portage.exception import InvalidData from portage.versions import _pkg_str -class IndexedVardb(object): +class IndexedVardb: """ A vardbapi interface that sacrifices validation in order to improve performance. It takes advantage of vardbdbapi._aux_cache, @@ -97,8 +97,7 @@ class IndexedVardb(object): if atom == atom.cp: return cp_list[:] - else: - return portage.match_from_list(atom, cp_list) + return portage.match_from_list(atom, cp_list) def aux_get(self, cpv, attrs, myrepo=None): pkg_data = self._vardb._aux_cache["packages"].get(cpv) diff --git a/lib/portage/dbapi/_ContentsCaseSensitivityManager.py b/lib/portage/dbapi/_ContentsCaseSensitivityManager.py index c479ec971..465d80554 100644 --- a/lib/portage/dbapi/_ContentsCaseSensitivityManager.py +++ b/lib/portage/dbapi/_ContentsCaseSensitivityManager.py @@ -1,7 +1,7 @@ # Copyright 2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -class ContentsCaseSensitivityManager(object): +class ContentsCaseSensitivityManager: """ Implicitly handles case transformations that are needed for case-insensitive support. diff --git a/lib/portage/dbapi/_MergeProcess.py b/lib/portage/dbapi/_MergeProcess.py index 371550079..e89b53555 100644 --- a/lib/portage/dbapi/_MergeProcess.py +++ b/lib/portage/dbapi/_MergeProcess.py @@ -1,11 +1,8 @@ -# Copyright 2010-2018 Gentoo Foundation +# Copyright 2010-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import io import platform -import signal -import sys -import traceback import fcntl import portage @@ -24,7 +21,7 @@ class MergeProcess(ForkProcess): 'vartree', 'blockers', 'pkgloc', 'infloc', 'myebuild', 'mydbapi', 'postinst_failure', 'prev_mtimes', 'unmerge', '_elog_reader_fd', - '_buf', '_elog_keys', '_locked_vdb') + '_buf', '_counter', '_dblink', '_elog_keys', '_locked_vdb') def _start(self): # Portage should always call setcpv prior to this @@ -57,6 +54,7 @@ class MergeProcess(ForkProcess): self.fd_pipes = self.fd_pipes.copy() self.fd_pipes.setdefault(0, portage._get_stdin().fileno()) + self.log_filter_file = self.settings.get('PORTAGE_LOG_FILTER_FILE_CMD') super(MergeProcess, self)._start() def _lock_vdb(self): @@ -102,8 +100,8 @@ class MergeProcess(ForkProcess): def _spawn(self, args, fd_pipes, **kwargs): """ - Fork a subprocess, apply local settings, and call - dblink.merge(). TODO: Share code with ForkProcess. + Extend the superclass _spawn method to perform some pre-fork and + post-fork actions. """ elog_reader_fd, elog_writer_fd = os.pipe() @@ -111,16 +109,6 @@ class MergeProcess(ForkProcess): fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL, fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK) - # FD_CLOEXEC is enabled by default in Python >=3.4. - if sys.hexversion < 0x3040000: - try: - fcntl.FD_CLOEXEC - except AttributeError: - pass - else: - fcntl.fcntl(elog_reader_fd, fcntl.F_SETFD, - fcntl.fcntl(elog_reader_fd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) - blockers = None if self.blockers is not None: # Query blockers in the main process, since closing @@ -141,57 +129,31 @@ class MergeProcess(ForkProcess): # FEATURES=parallel-install skips this lock in order to # improve performance, and the risk is practically negligible. self._lock_vdb() - counter = None if not self.unmerge: - counter = self.vartree.dbapi.counter_tick() - - parent_pid = os.getpid() - pid = None - try: - pid = os.fork() - - if pid != 0: - if not isinstance(pid, int): - raise AssertionError( - "fork returned non-integer: %s" % (repr(pid),)) - - os.close(elog_writer_fd) - self._elog_reader_fd = elog_reader_fd - self._buf = "" - self._elog_keys = set() - # Discard messages which will be collected by the subprocess, - # in order to avoid duplicates (bug #446136). - portage.elog.messages.collect_messages(key=mylink.mycpv) - - # invalidate relevant vardbapi caches - if self.vartree.dbapi._categories is not None: - self.vartree.dbapi._categories = None - self.vartree.dbapi._pkgs_changed = True - self.vartree.dbapi._clear_pkg_cache(mylink) - - return [pid] - - os.close(elog_reader_fd) - - # Use default signal handlers in order to avoid problems - # killing subprocesses as reported in bug #353239. - signal.signal(signal.SIGINT, signal.SIG_DFL) - signal.signal(signal.SIGTERM, signal.SIG_DFL) - - # Unregister SIGCHLD handler and wakeup_fd for the parent - # process's event loop (bug 655656). - signal.signal(signal.SIGCHLD, signal.SIG_DFL) - try: - wakeup_fd = signal.set_wakeup_fd(-1) - if wakeup_fd > 0: - os.close(wakeup_fd) - except (ValueError, OSError): - pass - - portage.locks._close_fds() - # We don't exec, so use close_fds=False - # (see _setup_pipes docstring). - portage.process._setup_pipes(fd_pipes, close_fds=False) + self._counter = self.vartree.dbapi.counter_tick() + + self._dblink = mylink + self._elog_reader_fd = elog_reader_fd + pids = super(MergeProcess, self)._spawn(args, fd_pipes, **kwargs) + os.close(elog_writer_fd) + self._buf = "" + self._elog_keys = set() + # Discard messages which will be collected by the subprocess, + # in order to avoid duplicates (bug #446136). + portage.elog.messages.collect_messages(key=mylink.mycpv) + + # invalidate relevant vardbapi caches + if self.vartree.dbapi._categories is not None: + self.vartree.dbapi._categories = None + self.vartree.dbapi._pkgs_changed = True + self.vartree.dbapi._clear_pkg_cache(mylink) + + return pids + + def _run(self): + os.close(self._elog_reader_fd) + counter = self._counter + mylink = self._dblink portage.output.havecolor = self.settings.get('NOCOLOR') \ not in ('yes', 'true') @@ -216,8 +178,7 @@ class MergeProcess(ForkProcess): self.settings.backup_changes("PORTAGE_BACKGROUND") rval = 1 - try: - if self.unmerge: + if self.unmerge: if not mylink.exists(): rval = os.EX_OK elif mylink.unmerge( @@ -228,37 +189,20 @@ class MergeProcess(ForkProcess): finally: mylink.unlockdb() rval = os.EX_OK - else: + else: rval = mylink.merge(self.pkgloc, self.infloc, myebuild=self.myebuild, mydbapi=self.mydbapi, prev_mtimes=self.prev_mtimes, counter=counter) - except SystemExit: - raise - except: - traceback.print_exc() - # os._exit() skips stderr flush! - sys.stderr.flush() - finally: - os._exit(rval) - - finally: - if pid == 0 or (pid is None and os.getpid() != parent_pid): - # Call os._exit() from a finally block in order - # to suppress any finally blocks from earlier - # in the call stack (see bug #345289). This - # finally block has to be setup before the fork - # in order to avoid a race condition. - os._exit(1) + return rval - def _async_waitpid_cb(self, *args, **kwargs): + def _proc_join_done(self, proc, future): """ - Override _async_waitpid_cb to perform cleanup that is - not necessarily idempotent. + Extend _proc_join_done to react to RETURNCODE_POSTINST_FAILURE. """ - ForkProcess._async_waitpid_cb(self, *args, **kwargs) - if self.returncode == portage.const.RETURNCODE_POSTINST_FAILURE: + if not future.cancelled() and proc.exitcode == portage.const.RETURNCODE_POSTINST_FAILURE: self.postinst_failure = True self.returncode = os.EX_OK + super(MergeProcess, self)._proc_join_done(proc, future) def _unregister(self): """ diff --git a/lib/portage/dbapi/_VdbMetadataDelta.py b/lib/portage/dbapi/_VdbMetadataDelta.py index 7461f87c5..ffdc0b361 100644 --- a/lib/portage/dbapi/_VdbMetadataDelta.py +++ b/lib/portage/dbapi/_VdbMetadataDelta.py @@ -10,7 +10,7 @@ from portage import _encodings from portage.util import atomic_ofstream from portage.versions import cpv_getkey -class VdbMetadataDelta(object): +class VdbMetadataDelta: _format_version = "1" diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py index 37728714e..bea5a82cb 100644 --- a/lib/portage/dbapi/__init__.py +++ b/lib/portage/dbapi/__init__.py @@ -1,8 +1,6 @@ # Copyright 1998-2019 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - __all__ = ["dbapi"] import functools @@ -26,7 +24,7 @@ from portage.exception import InvalidData from portage.localization import _ from _emerge.Package import Package -class dbapi(object): +class dbapi: _category_re = re.compile(r'^\w[-.+\w]*$', re.UNICODE) _categories = None _use_mutable = False diff --git a/lib/portage/dbapi/_expand_new_virt.py b/lib/portage/dbapi/_expand_new_virt.py index 9aa603d11..0edfb2ae2 100644 --- a/lib/portage/dbapi/_expand_new_virt.py +++ b/lib/portage/dbapi/_expand_new_virt.py @@ -1,8 +1,6 @@ # Copyright 2011-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - import portage from portage.dep import Atom, _get_useflag_re from portage.eapi import _get_eapi_attrs diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py index 2f6c4f343..9111cec06 100644 --- a/lib/portage/dbapi/bintree.py +++ b/lib/portage/dbapi/bintree.py @@ -1,8 +1,6 @@ -# Copyright 1998-2019 Gentoo Authors +# Copyright 1998-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - __all__ = ["bindbapi", "binarytree"] import portage @@ -46,7 +44,6 @@ import errno import io import stat import subprocess -import sys import tempfile import textwrap import time @@ -54,18 +51,8 @@ import traceback import warnings from gzip import GzipFile from itertools import chain -try: - from urllib.parse import urlparse -except ImportError: - from urlparse import urlparse - -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - _unicode = str - basestring = str - long = int -else: - _unicode = unicode +from urllib.parse import urlparse + class UseCachedCopyOfRemoteIndex(Exception): # If the local copy is recent enough @@ -152,7 +139,7 @@ class bindbapi(fakedbapi): add_pkg = self.bintree._additional_pkgs.get(instance_key) if add_pkg is not None: return add_pkg._db.aux_get(add_pkg, wants) - elif not self.bintree._remotepkgs or \ + if not self.bintree._remotepkgs or \ not self.bintree.isremote(mycpv): try: tbz2_path = self.bintree._pkg_paths[instance_key] @@ -166,9 +153,9 @@ class bindbapi(fakedbapi): metadata_bytes = portage.xpak.tbz2(tbz2_path).get_data() def getitem(k): if k == "_mtime_": - return _unicode(st[stat.ST_MTIME]) - elif k == "SIZE": - return _unicode(st.st_size) + return str(st[stat.ST_MTIME]) + if k == "SIZE": + return str(st.st_size) v = metadata_bytes.get(_unicode_encode(k, encoding=_encodings['repo.content'], errors='backslashreplace')) @@ -339,7 +326,7 @@ class bindbapi(fakedbapi): return filesdict -class binarytree(object): +class binarytree: "this tree scans for a list of all packages available in PKGDIR" def __init__(self, _unused=DeprecationWarning, pkgdir=None, virtual=DeprecationWarning, settings=None): @@ -471,7 +458,7 @@ class binarytree(object): # sanity check for atom in (origcp, newcp): if not isjustname(atom): - raise InvalidPackageName(_unicode(atom)) + raise InvalidPackageName(str(atom)) mynewcat = catsplit(newcp)[0] origmatches=self.dbapi.cp_list(origcp) moves = 0 @@ -495,7 +482,7 @@ class binarytree(object): if not isvalidatom(newcp, eapi=mycpv.eapi): continue - mynewcpv = mycpv.replace(mycpv_cp, _unicode(newcp), 1) + mynewcpv = mycpv.replace(mycpv_cp, str(newcp), 1) myoldpkg = catsplit(mycpv)[1] mynewpkg = catsplit(mynewcpv)[1] @@ -728,12 +715,12 @@ class binarytree(object): match = None for d in possibilities: try: - if long(d["_mtime_"]) != s[stat.ST_MTIME]: + if int(d["_mtime_"]) != s[stat.ST_MTIME]: continue except (KeyError, ValueError): continue try: - if long(d["SIZE"]) != long(s.st_size): + if int(d["SIZE"]) != int(s.st_size): continue except (KeyError, ValueError): continue @@ -820,7 +807,7 @@ class binarytree(object): if pkg_metadata.get("BUILD_ID"): try: - build_id = long(pkg_metadata["BUILD_ID"]) + build_id = int(pkg_metadata["BUILD_ID"]) except ValueError: writemsg(_("!!! Binary package has " "invalid BUILD_ID: '%s'\n") % @@ -850,8 +837,8 @@ class binarytree(object): noiselevel=-1) continue if build_id is not None: - pkg_metadata["BUILD_ID"] = _unicode(build_id) - pkg_metadata["SIZE"] = _unicode(s.st_size) + pkg_metadata["BUILD_ID"] = str(build_id) + pkg_metadata["SIZE"] = str(s.st_size) # Discard items used only for validation above. pkg_metadata.pop("CATEGORY") pkg_metadata.pop("PF") @@ -865,13 +852,13 @@ class binarytree(object): pkgindex._pkg_slot_dict()) if d: try: - if long(d["_mtime_"]) != s[stat.ST_MTIME]: + if int(d["_mtime_"]) != s[stat.ST_MTIME]: d.clear() except (KeyError, ValueError): d.clear() if d: try: - if long(d["SIZE"]) != long(s.st_size): + if int(d["SIZE"]) != int(s.st_size): d.clear() except (KeyError, ValueError): d.clear() @@ -1097,12 +1084,12 @@ class binarytree(object): writemsg(_("\n\n!!! Error fetching binhost package" \ " info from '%s'\n") % _hide_url_passwd(base_url)) # With Python 2, the EnvironmentError message may - # contain bytes or unicode, so use _unicode to ensure + # contain bytes or unicode, so use str to ensure # safety with all locales (bug #532784). try: - error_msg = _unicode(e) + error_msg = str(e) except UnicodeDecodeError as uerror: - error_msg = _unicode(uerror.object, + error_msg = str(uerror.object, encoding='utf_8', errors='replace') writemsg("!!! %s\n\n" % error_msg) del e @@ -1245,7 +1232,7 @@ class binarytree(object): # attributes, so that we can later distinguish that it # is identical to its remote counterpart. build_id = self._parse_build_id(basename) - metadata["BUILD_ID"] = _unicode(build_id) + metadata["BUILD_ID"] = str(build_id) cpv = _pkg_str(cpv, metadata=metadata, settings=self.settings, db=self.dbapi) binpkg = portage.xpak.tbz2(full_path) @@ -1297,9 +1284,9 @@ class binarytree(object): binary_metadata = portage.xpak.tbz2(filename).get_data() for k in keys: if k == "_mtime_": - metadata[k] = _unicode(st[stat.ST_MTIME]) + metadata[k] = str(st[stat.ST_MTIME]) elif k == "SIZE": - metadata[k] = _unicode(st.st_size) + metadata[k] = str(st.st_size) else: v = binary_metadata.get(_unicode_encode(k)) if v is None: @@ -1357,7 +1344,7 @@ class binarytree(object): contents = codecs.getwriter(_encodings['repo.content'])(io.BytesIO()) pkgindex.write(contents) contents = contents.getvalue() - atime = mtime = long(pkgindex.header["TIMESTAMP"]) + atime = mtime = int(pkgindex.header["TIMESTAMP"]) output_files = [(atomic_ofstream(self._pkgindex_file, mode="wb"), self._pkgindex_file, None)] @@ -1392,8 +1379,8 @@ class binarytree(object): d["CPV"] = cpv st = os.lstat(pkg_path) - d["_mtime_"] = _unicode(st[stat.ST_MTIME]) - d["SIZE"] = _unicode(st.st_size) + d["_mtime_"] = str(st[stat.ST_MTIME]) + d["SIZE"] = str(st.st_size) rel_path = pkg_path[len(self.pkgdir)+1:] # record location if it's non-default @@ -1472,7 +1459,7 @@ class binarytree(object): """ if not (self.settings.profile_path and "IUSE_IMPLICIT" in self.settings): - header.setdefault("VERSION", _unicode(self._pkgindex_version)) + header.setdefault("VERSION", str(self._pkgindex_version)) return portdir = normalize_path(os.path.realpath(self.settings["PORTDIR"])) @@ -1483,7 +1470,7 @@ class binarytree(object): if profile_path.startswith(profiles_base): profile_path = profile_path[len(profiles_base):] header["PROFILE"] = profile_path - header["VERSION"] = _unicode(self._pkgindex_version) + header["VERSION"] = str(self._pkgindex_version) base_uri = self.settings.get("PORTAGE_BINHOST_HEADER_URI") if base_uri: header["URI"] = base_uri @@ -1658,7 +1645,7 @@ class binarytree(object): if hyphen != -1: build_id = filename[hyphen+1:-suffixlen] try: - build_id = long(build_id) + build_id = int(build_id) except ValueError: pass return build_id @@ -1671,7 +1658,7 @@ class binarytree(object): instance_key = self.dbapi._instance_key(pkgname) if instance_key not in self._remotepkgs: return False - elif instance_key in self._additional_pkgs: + if instance_key in self._additional_pkgs: return False # Presence in self._remotepkgs implies that it's remote. When a # package is downloaded, state is updated by self.inject(). @@ -1696,10 +1683,10 @@ class binarytree(object): if os.path.exists(tbz2_path): if tbz2name[:-5] not in self.invalids: return - else: - resume = True - writemsg(_("Resuming download of this tbz2, but it is possible that it is corrupt.\n"), - noiselevel=-1) + + resume = True + writemsg(_("Resuming download of this tbz2, but it is possible that it is corrupt.\n"), + noiselevel=-1) mydest = os.path.dirname(self.getname(pkgname)) self._ensure_dir(mydest) diff --git a/lib/portage/dbapi/cpv_expand.py b/lib/portage/dbapi/cpv_expand.py index ac2f6cc2e..a1a91f554 100644 --- a/lib/portage/dbapi/cpv_expand.py +++ b/lib/portage/dbapi/cpv_expand.py @@ -1,8 +1,6 @@ # Copyright 2010-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - __all__ = ["cpv_expand"] import portage @@ -102,7 +100,5 @@ def cpv_expand(mycpv, mydb=None, use_cache=1, settings=None): if mysplit: if mysplit[2]=="r0": return mykey+"-"+mysplit[1] - else: - return mykey+"-"+mysplit[1]+"-"+mysplit[2] - else: - return mykey + return mykey+"-"+mysplit[1]+"-"+mysplit[2] + return mykey diff --git a/lib/portage/dbapi/dep_expand.py b/lib/portage/dbapi/dep_expand.py index 9515b7dec..fab5fa830 100644 --- a/lib/portage/dbapi/dep_expand.py +++ b/lib/portage/dbapi/dep_expand.py @@ -1,8 +1,6 @@ # Copyright 2010-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - __all__ = ["dep_expand"] import re diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py index 08af17bcd..4b714a919 100644 --- a/lib/portage/dbapi/porttree.py +++ b/lib/portage/dbapi/porttree.py @@ -1,8 +1,6 @@ # Copyright 1998-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - __all__ = [ "close_portdbapi_caches", "FetchlistDict", "portagetree", "portdbapi" ] @@ -41,7 +39,6 @@ from portage.util.futures.iter_completed import iter_gather from _emerge.EbuildMetadataPhase import EbuildMetadataPhase import os as _os -import sys import traceback import warnings import errno @@ -49,16 +46,8 @@ import collections import functools from collections import OrderedDict +from urllib.parse import urlparse -try: - from urllib.parse import urlparse -except ImportError: - from urlparse import urlparse - -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str - long = int def close_portdbapi_caches(): # The python interpreter does _not_ guarantee that destructors are @@ -108,7 +97,7 @@ class _dummy_list(list): pass -class _better_cache(object): +class _better_cache: """ The purpose of better_cache is to locate catpkgs in repositories using ``os.listdir()`` as much as possible, which @@ -164,8 +153,13 @@ class _better_cache(object): raise continue for p in pkg_list: - if os.path.isdir(cat_dir + "/" + p): - self._items[cat + "/" + p].append(repo) + try: + atom = Atom("%s/%s" % (cat, p)) + except InvalidAtom: + continue + if atom != atom.cp: + continue + self._items[atom.cp].append(repo) self._scanned_cats.add(cat) @@ -883,7 +877,7 @@ class portdbapi(dbapi): filesdict[myfile] = int(checksums[myfile]["size"]) return filesdict - def fetch_check(self, mypkg, useflags=None, mysettings=None, all=False, myrepo=None): + def fetch_check(self, mypkg, useflags=None, mysettings=None, all=False, myrepo=None): # pylint: disable=redefined-builtin """ TODO: account for PORTAGE_RO_DISTDIRS """ @@ -936,8 +930,7 @@ class portdbapi(dbapi): return 0 if self.findname(cps[0] + "/" + cps2[1], myrepo=myrepo): return 1 - else: - return 0 + return 0 def cp_all(self, categories=None, trees=None, reverse=False, sort=True): """ @@ -994,7 +987,7 @@ class portdbapi(dbapi): # stable sort by version produces results ordered by # (pkg.version, repo.priority). if mytree is not None: - if isinstance(mytree, basestring): + if isinstance(mytree, str): repos = [self.repositories.get_repo_for_location(mytree)] else: # assume it's iterable @@ -1316,7 +1309,7 @@ class portdbapi(dbapi): return True -class portagetree(object): +class portagetree: def __init__(self, root=DeprecationWarning, virtual=DeprecationWarning, settings=None): """ @@ -1456,12 +1449,7 @@ class FetchlistDict(Mapping): infinite recursion in some cases.""" return len(self.portdb.cp_list(self.cp, mytree=self.mytree)) - def keys(self): - """Returns keys for all packages within pkgdir""" - return self.portdb.cp_list(self.cp, mytree=self.mytree) - - if sys.hexversion >= 0x3000000: - keys = __iter__ + keys = __iter__ def _async_manifest_fetchlist(portdb, repo_config, cp, cpv_list=None, @@ -1508,7 +1496,7 @@ def _async_manifest_fetchlist(portdb, repo_config, cp, cpv_list=None, if result.cancelled(): return - elif e is None: + if e is None: result.set_result(dict((k, list(v.result())) for k, v in zip(cpv_list, gather_result.result()))) else: diff --git a/lib/portage/dbapi/vartree.py b/lib/portage/dbapi/vartree.py index d7c8ef3de..4a45a3169 100644 --- a/lib/portage/dbapi/vartree.py +++ b/lib/portage/dbapi/vartree.py @@ -1,7 +1,7 @@ -# Copyright 1998-2019 Gentoo Authors +# Copyright 1998-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import division, unicode_literals +from __future__ import division __all__ = [ "vardbapi", "vartree", "dblink"] + \ @@ -36,7 +36,6 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.util.install_mask:install_mask_dir,InstallMask,_raise_exc', 'portage.util.listdir:dircache,listdir', 'portage.util.movefile:movefile', - 'portage.util.monotonic:monotonic', 'portage.util.path:first_existing,iter_parents', 'portage.util.writeable_check:get_ro_checker', 'portage.util._xattr:xattr', @@ -96,28 +95,16 @@ from itertools import chain import logging import os as _os import operator +import pickle import platform import pwd import re import stat -import sys import tempfile import textwrap import time import warnings -try: - import cPickle as pickle -except ImportError: - import pickle - -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str - long = int - _unicode = str -else: - _unicode = unicode class vardbapi(dbapi): @@ -366,7 +353,7 @@ class vardbapi(dbapi): def cpv_counter(self, mycpv): "This method will grab the COUNTER. Returns a counter value." try: - return long(self.aux_get(mycpv, ["COUNTER"])[0]) + return int(self.aux_get(mycpv, ["COUNTER"])[0]) except (KeyError, ValueError): pass writemsg_level(_("portage: COUNTER for %s was corrupted; " \ @@ -419,7 +406,7 @@ class vardbapi(dbapi): if not isvalidatom(newcp, eapi=mycpv.eapi): continue - mynewcpv = mycpv.replace(mycpv_cp, _unicode(newcp), 1) + mynewcpv = mycpv.replace(mycpv_cp, str(newcp), 1) mynewcat = catsplit(newcp)[0] origpath = self.getpath(mycpv) if not os.path.exists(origpath): @@ -457,10 +444,7 @@ class vardbapi(dbapi): if mysplit[0] == '*': mysplit[0] = mysplit[0][1:] try: - if sys.hexversion >= 0x3030000: - mystat = os.stat(self.getpath(mysplit[0])).st_mtime_ns - else: - mystat = os.stat(self.getpath(mysplit[0])).st_mtime + mystat = os.stat(self.getpath(mysplit[0])).st_mtime_ns except OSError: mystat = 0 if use_cache and mycp in self.cpcache: @@ -608,10 +592,7 @@ class vardbapi(dbapi): return list(self._iter_match(mydep, self.cp_list(mydep.cp, use_cache=use_cache))) try: - if sys.hexversion >= 0x3030000: - curmtime = os.stat(os.path.join(self._eroot, VDB_PATH, mycat)).st_mtime_ns - else: - curmtime = os.stat(os.path.join(self._eroot, VDB_PATH, mycat)).st_mtime + curmtime = os.stat(os.path.join(self._eroot, VDB_PATH, mycat)).st_mtime_ns except (IOError, OSError): curmtime=0 @@ -774,7 +755,7 @@ class vardbapi(dbapi): pkg_data = None else: cache_mtime, metadata = pkg_data - if not isinstance(cache_mtime, (float, long, int)) or \ + if not isinstance(cache_mtime, (float, int)) or \ not isinstance(metadata, dict): pkg_data = None @@ -786,7 +767,7 @@ class vardbapi(dbapi): # Handle truncated mtime in order to avoid cache # invalidation for livecd squashfs (bug 564222). - elif long(cache_mtime) == mydir_stat.st_mtime: + elif int(cache_mtime) == mydir_stat.st_mtime: cache_valid = True else: # Cache may contain integer mtime. @@ -811,7 +792,7 @@ class vardbapi(dbapi): cache_data.update(metadata) for aux_key in cache_these: cache_data[aux_key] = mydata[aux_key] - self._aux_cache["packages"][_unicode(mycpv)] = \ + self._aux_cache["packages"][str(mycpv)] = \ (mydir_mtime, cache_data) self._aux_cache["modified"].add(mycpv) @@ -1105,7 +1086,7 @@ class vardbapi(dbapi): mode='r', encoding=_encodings['repo.content'], errors='replace') as f: try: - counter = long(f.readline().strip()) + counter = int(f.readline().strip()) except (OverflowError, ValueError) as e: writemsg(_("!!! COUNTER file is corrupt: '%s'\n") % self._counter_path, noiselevel=-1) @@ -1264,7 +1245,7 @@ class vardbapi(dbapi): if new_needed is not None: f = atomic_ofstream(os.path.join(pkg.dbdir, LinkageMap._needed_aux_key)) for entry in new_needed: - f.write(_unicode(entry)) + f.write(str(entry)) f.close() f = atomic_ofstream(os.path.join(pkg.dbdir, "CONTENTS")) write_contents(new_contents, root, f) @@ -1272,7 +1253,7 @@ class vardbapi(dbapi): self._bump_mtime(pkg.mycpv) pkg._clear_contents_cache() - class _owners_cache(object): + class _owners_cache: """ This class maintains an hash table that serves to index package contents by mapping the basename of file to a list of possible @@ -1336,9 +1317,9 @@ class vardbapi(dbapi): counter = int(counter) except ValueError: counter = 0 - return (_unicode(cpv), counter, mtime) + return (str(cpv), counter, mtime) - class _owners_db(object): + class _owners_db: def __init__(self, vardb): self._vardb = vardb @@ -1465,7 +1446,7 @@ class vardbapi(dbapi): len(hash_value) != 3: continue cpv, counter, mtime = hash_value - if not isinstance(cpv, basestring): + if not isinstance(cpv, str): continue try: current_hash = hash_pkg(cpv) @@ -1552,7 +1533,7 @@ class vardbapi(dbapi): for result in search_future.result(): yield result -class vartree(object): +class vartree: "this tree will scan a var/db/pkg database located at root (passed to init)" def __init__(self, root=None, virtual=DeprecationWarning, categories=None, settings=None): @@ -1609,8 +1590,7 @@ class vartree(object): use_cache=use_cache)) if mymatch is None: return "" - else: - return mymatch + return mymatch def dep_match(self, mydep, use_cache=1): "compatibility method -- we want to see all matches, not just visible ones" @@ -1618,8 +1598,7 @@ class vartree(object): mymatch = self.dbapi.match(mydep, use_cache=use_cache) if mymatch is None: return [] - else: - return mymatch + return mymatch def exists_specific(self, cpv): return self.dbapi.cpv_exists(cpv) @@ -1648,7 +1627,7 @@ class vartree(object): def populate(self): self.populated=1 -class dblink(object): +class dblink: """ This class provides an interface to the installed package database At present this is implemented as a text backend in /var/db/pkg. @@ -3638,7 +3617,7 @@ class dblink(object): symlink_collisions = [] destroot = self.settings['ROOT'] totfiles = len(file_list) + len(symlink_list) - previous = monotonic() + previous = time.monotonic() progress_shown = False report_interval = 1.7 # seconds falign = len("%d" % totfiles) @@ -3647,7 +3626,7 @@ class dblink(object): for i, (f, f_type) in enumerate(chain( ((f, "reg") for f in file_list), ((f, "sym") for f in symlink_list))): - current = monotonic() + current = time.monotonic() if current - previous > report_interval: showMessage(_("%3d%% done, %*d files remaining ...\n") % (i * 100 / totfiles, falign, totfiles - i)) @@ -3913,7 +3892,7 @@ class dblink(object): for phase, messages in logentries.items(): for key, lines in messages: funcname = funcnames[key] - if isinstance(lines, basestring): + if isinstance(lines, str): lines = [lines] for line in lines: for line in line.split('\n'): @@ -4931,7 +4910,7 @@ class dblink(object): self._installed_instance is not None # this is supposed to merge a list of files. There will be 2 forms of argument passing. - if isinstance(stufftomerge, basestring): + if isinstance(stufftomerge, str): #A directory is specified. Figure out protection paths, listdir() it and process it. mergelist = [join(stufftomerge, child) for child in \ os.listdir(join(srcroot, stufftomerge))] @@ -4951,10 +4930,7 @@ class dblink(object): mymd5 = None myto = None - if sys.hexversion >= 0x3030000: - mymtime = mystat.st_mtime_ns - else: - mymtime = mystat[stat.ST_MTIME] + mymtime = mystat.st_mtime_ns if stat.S_ISREG(mymode): mymd5 = perform_md5(mysrc, calc_prelink=calc_prelink) @@ -5097,10 +5073,7 @@ class dblink(object): % (relative_path, myabsto)]) showMessage("%s %s -> %s\n" % (zing, mydest, myto)) - if sys.hexversion >= 0x3030000: - outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime // 1000000000)+"\n") - else: - outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime)+"\n") + outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime // 1000000000)+"\n") else: showMessage(_("!!! Failed to move file.\n"), level=logging.ERROR, noiselevel=-1) @@ -5249,10 +5222,7 @@ class dblink(object): pass if mymtime != None: - if sys.hexversion >= 0x3030000: - outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime // 1000000000)+"\n") - else: - outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime)+"\n") + outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime // 1000000000)+"\n") showMessage("%s %s\n" % (zing,mydest)) else: # we are merging a fifo or device node @@ -5470,7 +5440,7 @@ class dblink(object): def setfile(self,fname,data): kwargs = {} - if fname == 'environment.bz2' or not isinstance(data, basestring): + if fname == 'environment.bz2' or not isinstance(data, str): kwargs['mode'] = 'wb' else: kwargs['mode'] = 'w' @@ -5527,7 +5497,7 @@ class dblink(object): build_time = backup_dblink.getfile('BUILD_TIME') try: - build_time = long(build_time.strip()) + build_time = int(build_time.strip()) except ValueError: build_time = 0 diff --git a/lib/portage/dbapi/virtual.py b/lib/portage/dbapi/virtual.py index 3f7e6c221..f62fc2a30 100644 --- a/lib/portage/dbapi/virtual.py +++ b/lib/portage/dbapi/virtual.py @@ -1,8 +1,6 @@ # Copyright 1998-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - from portage.dbapi import dbapi from portage.dbapi.dep_expand import dep_expand from portage.versions import cpv_getkey, _pkg_str @@ -125,8 +123,7 @@ class fakedbapi(dbapi): def cpv_all(self): if self._multi_instance: return [x[0] for x in self.cpvdict] - else: - return list(self.cpvdict) + return list(self.cpvdict) def cpv_inject(self, mycpv, metadata=None): """Adds a cpv to the list of available packages. See the @@ -215,7 +212,7 @@ class fakedbapi(dbapi): raise KeyError(cpv) metadata.update(values) -class testdbapi(object): +class testdbapi: """A dbapi instance with completely fake functions to get by hitting disk TODO(antarus): This class really needs to be rewritten to have better stubs; but these work for now. diff --git a/lib/portage/debug.py b/lib/portage/debug.py index 193e62291..4db9da53b 100644 --- a/lib/portage/debug.py +++ b/lib/portage/debug.py @@ -21,7 +21,7 @@ def set_trace(on=True): sys.settrace(None) threading.settrace(None) -class trace_handler(object): +class trace_handler: def __init__(self): python_system_paths = [] @@ -63,7 +63,7 @@ class trace_handler(object): if len(my_repr) > self.max_repr_length: my_repr = "'omitted'" return "value=%s " % my_repr - elif "exception" == event: + if "exception" == event: my_repr = repr(arg[1]) if len(my_repr) > self.max_repr_length: my_repr = "'omitted'" @@ -100,7 +100,7 @@ class trace_handler(object): my_locals[k] = "omitted" return my_locals -class prefix_trimmer(object): +class prefix_trimmer: def __init__(self, prefix): self.prefix = prefix self.cut_index = len(prefix) @@ -112,9 +112,9 @@ class prefix_trimmer(object): The previous result is automatically cached.""" if s == self.previous: return self.previous_trimmed + + if s.startswith(self.prefix): + self.previous_trimmed = s[self.cut_index:] else: - if s.startswith(self.prefix): - self.previous_trimmed = s[self.cut_index:] - else: - self.previous_trimmed = s - return self.previous_trimmed + self.previous_trimmed = s + return self.previous_trimmed diff --git a/lib/portage/dep/__init__.py b/lib/portage/dep/__init__.py index 72988357a..36f0dc8c9 100644 --- a/lib/portage/dep/__init__.py +++ b/lib/portage/dep/__init__.py @@ -2,8 +2,6 @@ # Copyright 2003-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - __all__ = [ 'Atom', 'best_match_to_list', 'cpvequal', 'dep_getcpv', 'dep_getkey', 'dep_getslot', @@ -17,6 +15,7 @@ __all__ = [ import re, sys import warnings +from functools import lru_cache import portage portage.proxy.lazyimport.lazyimport(globals(), @@ -31,12 +30,6 @@ from portage.versions import _cp, _cpv, _pkg_str, _slot, _unknown_repo, _vr, \ catpkgsplit, vercmp, ververify import portage.cache.mappings -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str - _unicode = str -else: - _unicode = unicode # \w is [a-zA-Z0-9_] @@ -78,7 +71,7 @@ def _match_slot(atom, pkg): if pkg.slot == atom.slot: if not atom.sub_slot: return True - elif atom.sub_slot == pkg.sub_slot: + if atom.sub_slot == pkg.sub_slot: return True return False @@ -353,7 +346,7 @@ class paren_normalize(list): return dest i = iter(src) for x in i: - if isinstance(x, basestring): + if isinstance(x, str): if x in ('||', '^^'): y = self._zap_parens(next(i), [], disjunction=True) if len(y) == 1: @@ -404,49 +397,10 @@ def paren_enclose(mylist, unevaluated_atom=False, opconvert=False): mystrparts.append(x) return " ".join(mystrparts) -def use_reduce(depstr, uselist=(), masklist=(), matchall=False, excludeall=(), is_src_uri=False, \ - eapi=None, opconvert=False, flat=False, is_valid_flag=None, token_class=None, matchnone=False, - subset=None): - """ - Takes a dep string and reduces the use? conditionals out, leaving an array - with subarrays. All redundant brackets are removed. - - @param depstr: depstring - @type depstr: String - @param uselist: Sequence of use enabled flags - @type uselist: Sequence - @param masklist: Sequence of masked flags (always treated as disabled) - @type masklist: Sequence - @param matchall: Treat all conditionals as active. Used by repoman. - @type matchall: Bool - @param excludeall: Sequence of flags for which negated conditionals are always treated as inactive. - @type excludeall: Sequence - @param is_src_uri: Indicates if depstr represents a SRC_URI - @type is_src_uri: Bool - @param eapi: Indicates the EAPI the dep string has to comply to - @type eapi: String - @param opconvert: Put every operator as first element into it's argument list - @type opconvert: Bool - @param flat: Create a flat list of all tokens - @type flat: Bool - @param is_valid_flag: Function that decides if a given use flag might be used in use conditionals - @type is_valid_flag: Function - @param token_class: Convert all non operator tokens into this class - @type token_class: Class - @param matchnone: Treat all conditionals as inactive. Used by digestgen(). - @type matchnone: Bool - @param subset: Select a subset of dependencies conditional on the given flags - @type subset: Sequence - @rtype: List - @return: The use reduced depend array - """ - if isinstance(depstr, list): - if portage._internal_caller: - warnings.warn(_("Passing paren_reduced dep arrays to %s is deprecated. " + \ - "Pass the original dep string instead.") % \ - ('portage.dep.use_reduce',), DeprecationWarning, stacklevel=2) - depstr = paren_enclose(depstr) - +@lru_cache(1024) +def _use_reduce_cached(depstr, uselist, masklist, matchall, excludeall, \ + is_src_uri, eapi, opconvert, flat, is_valid_flag, token_class, \ + matchnone,subset): if opconvert and flat: raise ValueError("portage.dep.use_reduce: 'opconvert' and 'flat' are mutually exclusive") @@ -597,13 +551,12 @@ def use_reduce(depstr, uselist=(), masklist=(), matchall=False, excludeall=(), i stack[level].extend(l) continue - if stack[level] and isinstance(stack[level][-1], - basestring): + if stack[level] and isinstance(stack[level][-1], str): if stack[level][-1] == "||" and not l: #Optimize: || ( ) -> . if not eapi_attrs.empty_groups_always_true: # in EAPI 7+, we need to fail here - l.append((token_class or _unicode)("__const__/empty-any-of")) + l.append((token_class or str)("__const__/empty-any-of")) stack[level].pop() elif stack[level][-1][-1] == "?": #The last token before the '(' that matches the current ')' @@ -626,11 +579,10 @@ def use_reduce(depstr, uselist=(), masklist=(), matchall=False, excludeall=(), i #ends in a non-operator. This is almost equivalent to stack[level][-1]=="||", #expect that it skips empty levels. while k>=0: - if stack[k] and isinstance(stack[k][-1], - basestring): + if stack[k] and isinstance(stack[k][-1], str): if stack[k][-1] == "||": return k - elif stack[k][-1][-1] != "?": + if stack[k][-1][-1] != "?": return -1 k -= 1 return -1 @@ -769,6 +721,65 @@ def use_reduce(depstr, uselist=(), masklist=(), matchall=False, excludeall=(), i return stack[0] +def use_reduce(depstr, uselist=(), masklist=(), matchall=False, excludeall=(), is_src_uri=False, \ + eapi=None, opconvert=False, flat=False, is_valid_flag=None, token_class=None, matchnone=False, + subset=None): + """ + Takes a dep string and reduces the use? conditionals out, leaving an array + with subarrays. All redundant brackets are removed. + + @param depstr: depstring + @type depstr: String + @param uselist: Sequence of use enabled flags + @type uselist: Sequence + @param masklist: Sequence of masked flags (always treated as disabled) + @type masklist: Sequence + @param matchall: Treat all conditionals as active. Used by repoman. + @type matchall: Bool + @param excludeall: Sequence of flags for which negated conditionals are always treated as inactive. + @type excludeall: Sequence + @param is_src_uri: Indicates if depstr represents a SRC_URI + @type is_src_uri: Bool + @param eapi: Indicates the EAPI the dep string has to comply to + @type eapi: String + @param opconvert: Put every operator as first element into it's argument list + @type opconvert: Bool + @param flat: Create a flat list of all tokens + @type flat: Bool + @param is_valid_flag: Function that decides if a given use flag might be used in use conditionals + @type is_valid_flag: Function + @param token_class: Convert all non operator tokens into this class + @type token_class: Class + @param matchnone: Treat all conditionals as inactive. Used by digestgen(). + @type matchnone: Bool + @param subset: Select a subset of dependencies conditional on the given flags + @type subset: Sequence + @rtype: List + @return: The use reduced depend array + """ + if isinstance(depstr, list): + if portage._internal_caller: + warnings.warn(_("Passing paren_reduced dep arrays to %s is deprecated. " + \ + "Pass the original dep string instead.") % \ + ('portage.dep.use_reduce',), DeprecationWarning, stacklevel=2) + depstr = paren_enclose(depstr) + + if uselist is not None: + uselist = frozenset(uselist) + if masklist is not None: + masklist = frozenset(masklist) + if excludeall is not None: + excludeall = frozenset(excludeall) + if subset is not None: + subset = frozenset(subset) + + result = _use_reduce_cached(depstr, uselist, masklist, matchall, \ + excludeall, is_src_uri, eapi, opconvert, flat, is_valid_flag, \ + token_class, matchnone, subset) + + # The list returned by this function may be modified, so return a copy. + return result[:] + def dep_opconvert(deplist): """ Iterate recursively through a list of deps, if the @@ -829,12 +840,12 @@ def flatten(mylist): newlist.append(x) return newlist -class _use_dep(object): +class _use_dep: __slots__ = ("_eapi_attrs", "conditional", "missing_enabled", "missing_disabled", "disabled", "enabled", "tokens", "required") - class _conditionals_class(object): + class _conditionals_class: __slots__ = ("enabled", "disabled", "equal", "not_equal") def items(self): @@ -949,22 +960,11 @@ class _use_dep(object): def __bool__(self): return bool(self.tokens) - if sys.hexversion < 0x3000000: - __nonzero__ = __bool__ - def __str__(self): if not self.tokens: return "" return "[%s]" % (",".join(self.tokens),) - if sys.hexversion < 0x3000000: - - __unicode__ = __str__ - - def __str__(self): - return _unicode_encode(self.__unicode__(), - encoding=_encodings['content'], errors='backslashreplace') - def __repr__(self): return "portage.dep._use_dep(%s)" % repr(self.tokens) @@ -1209,7 +1209,7 @@ class _use_dep(object): return _use_dep(tokens, self._eapi_attrs, enabled_flags=enabled_flags, disabled_flags=disabled_flags, missing_enabled=missing_enabled, missing_disabled=missing_disabled, required=self.required) -class Atom(_unicode): +class Atom(str): """ For compatibility with existing atom string manipulation code, this @@ -1222,10 +1222,10 @@ class Atom(_unicode): # Distiguishes soname atoms from other atom types soname = False - class _blocker(object): + class _blocker: __slots__ = ("overlap",) - class _overlap(object): + class _overlap: __slots__ = ("forbid",) def __init__(self, forbid=False): @@ -1236,7 +1236,7 @@ class Atom(_unicode): def __new__(cls, s, unevaluated_atom=None, allow_wildcard=False, allow_repo=None, _use=None, eapi=None, is_valid_flag=None, allow_build_id=None): - return _unicode.__new__(cls, s) + return str.__new__(cls, s) def __init__(self, s, unevaluated_atom=None, allow_wildcard=False, allow_repo=None, _use=None, eapi=None, is_valid_flag=None, allow_build_id=None): @@ -1244,13 +1244,13 @@ class Atom(_unicode): # This is an efficiency assertion, to ensure that the Atom # constructor is not called redundantly. raise TypeError(_("Expected %s, got %s") % \ - (_unicode, type(s))) + (str, type(s))) - if not isinstance(s, _unicode): - # Avoid TypeError from _unicode.__init__ with PyPy. + if not isinstance(s, str): + # Avoid TypeError from str.__init__ with PyPy. s = _unicode_decode(s) - _unicode.__init__(s) + str.__init__(s) eapi_attrs = _get_eapi_attrs(eapi) atom_re = _get_atom_re(eapi_attrs) @@ -1417,7 +1417,7 @@ class Atom(_unicode): unevaluated_atom.use is not None: # unevaluated_atom.use is used for IUSE checks when matching # packages, so it must not propagate to without_use - without_use = Atom(_unicode(self), + without_use = Atom(str(self), allow_wildcard=allow_wildcard, allow_repo=allow_repo, eapi=eapi) @@ -1433,9 +1433,9 @@ class Atom(_unicode): self.__dict__['unevaluated_atom'] = self if eapi is not None: - if not isinstance(eapi, basestring): + if not isinstance(eapi, str): raise TypeError('expected eapi argument of ' + \ - '%s, got %s: %s' % (basestring, type(eapi), eapi,)) + '%s, got %s: %s' % (str, type(eapi), eapi,)) if self.slot and not eapi_attrs.slot_deps: raise InvalidAtom( _("Slot deps are not allowed in EAPI %s: '%s'") \ @@ -1497,7 +1497,7 @@ class Atom(_unicode): if self.repo is not None: atom += _repo_separator + self.repo if self.use is not None: - atom += _unicode(self.use) + atom += str(self.use) return Atom(atom, allow_repo=True, allow_wildcard=True) @@ -1513,7 +1513,7 @@ class Atom(_unicode): atom += self.slot_operator atom += _repo_separator + repo if self.use is not None: - atom += _unicode(self.use) + atom += str(self.use) return Atom(atom, allow_repo=True, allow_wildcard=True) def with_slot(self, slot): @@ -1521,7 +1521,7 @@ class Atom(_unicode): if self.repo is not None: atom += _repo_separator + self.repo if self.use is not None: - atom += _unicode(self.use) + atom += str(self.use) return Atom(atom, allow_repo=True, allow_wildcard=True) def __setattr__(self, name, value): @@ -1579,7 +1579,7 @@ class Atom(_unicode): if self.slot_operator is not None: atom += self.slot_operator use_dep = self.use.evaluate_conditionals(use) - atom += _unicode(use_dep) + atom += str(use_dep) return Atom(atom, unevaluated_atom=self, allow_repo=(self.repo is not None), _use=use_dep) def violated_conditionals(self, other_use, is_valid_flag, parent_use=None): @@ -1607,7 +1607,7 @@ class Atom(_unicode): if self.slot_operator is not None: atom += self.slot_operator use_dep = self.use.violated_conditionals(other_use, is_valid_flag, parent_use) - atom += _unicode(use_dep) + atom += str(use_dep) return Atom(atom, unevaluated_atom=self, allow_repo=(self.repo is not None), _use=use_dep) def _eval_qa_conditionals(self, use_mask, use_force): @@ -1623,7 +1623,7 @@ class Atom(_unicode): if self.slot_operator is not None: atom += self.slot_operator use_dep = self.use._eval_qa_conditionals(use_mask, use_force) - atom += _unicode(use_dep) + atom += str(use_dep) return Atom(atom, unevaluated_atom=self, allow_repo=(self.repo is not None), _use=use_dep) def __copy__(self): @@ -1701,12 +1701,10 @@ class ExtendedAtomDict(portage.cache.mappings.MutableMapping): def __delitem__(self, cp): if "*" in cp: return self._extended.__delitem__(cp) - else: - return self._normal.__delitem__(cp) + return self._normal.__delitem__(cp) - if sys.hexversion >= 0x3000000: - keys = __iter__ - items = iteritems + keys = __iter__ + items = iteritems def __len__(self): return len(self._normal) + len(self._extended) @@ -1714,12 +1712,11 @@ class ExtendedAtomDict(portage.cache.mappings.MutableMapping): def setdefault(self, cp, default=None): if "*" in cp: return self._extended.setdefault(cp, default) - else: - return self._normal.setdefault(cp, default) + return self._normal.setdefault(cp, default) def __getitem__(self, cp): - if not isinstance(cp, basestring): + if not isinstance(cp, str): raise KeyError(cp) if '*' in cp: @@ -1832,8 +1829,7 @@ def dep_getslot(mydep): bracket = mydep.find("[", colon) if bracket == -1: return mydep[colon+1:] - else: - return mydep[colon+1:bracket] + return mydep[colon+1:bracket] return None def dep_getrepo(mydep): @@ -1864,8 +1860,7 @@ def dep_getrepo(mydep): bracket = mydep.find("[", colon) if bracket == -1: return mydep[colon+2:] - else: - return mydep[colon+2:bracket] + return mydep[colon+2:bracket] return None def remove_slot(mydep): @@ -1961,7 +1956,7 @@ def isvalidatom(atom, allow_blockers=False, allow_wildcard=False, if eapi is not None and isinstance(atom, Atom) and atom.eapi != eapi: # We'll construct a new atom with the given eapi. - atom = _unicode(atom) + atom = str(atom) try: if not isinstance(atom, Atom): @@ -2509,7 +2504,7 @@ def get_required_use_flags(required_use, eapi=None): return frozenset(used_flags) -class _RequiredUseLeaf(object): +class _RequiredUseLeaf: __slots__ = ('_satisfied', '_token') @@ -2520,7 +2515,7 @@ class _RequiredUseLeaf(object): def tounicode(self): return self._token -class _RequiredUseBranch(object): +class _RequiredUseBranch: __slots__ = ('_children', '_operator', '_parent', '_satisfied') @@ -2564,8 +2559,6 @@ class _RequiredUseBranch(object): return " ".join(tokens) - if sys.hexversion < 0x3000000: - __nonzero__ = __bool__ def check_required_use(required_use, use, iuse_match, eapi=None): """ @@ -2618,11 +2611,11 @@ def check_required_use(required_use, use, iuse_match, eapi=None): if operator == "||": return (True in argument) - elif operator == "^^": + if operator == "^^": return (argument.count(True) == 1) - elif operator == "??": + if operator == "??": return (argument.count(True) <= 1) - elif operator[-1] == "?": + if operator[-1] == "?": return (False not in argument) mysplit = required_use.split() diff --git a/lib/portage/dep/_dnf.py b/lib/portage/dep/_dnf.py index 59657fd6a..1b14d2b43 100644 --- a/lib/portage/dep/_dnf.py +++ b/lib/portage/dep/_dnf.py @@ -1,8 +1,6 @@ # Copyright 2017 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - import itertools @@ -85,6 +83,6 @@ def contains_disjunction(dep_struct): assert x, 'Normalization error, empty conjunction found in %s' % (dep_struct,) if x[0] == '||': return True - elif is_disjunction and contains_disjunction(x): + if is_disjunction and contains_disjunction(x): return True return False diff --git a/lib/portage/dep/_slot_operator.py b/lib/portage/dep/_slot_operator.py index a9f74382c..29bf8e1b1 100644 --- a/lib/portage/dep/_slot_operator.py +++ b/lib/portage/dep/_slot_operator.py @@ -1,8 +1,6 @@ # Copyright 2012-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - from portage.dep import Atom, paren_enclose, use_reduce from portage.eapi import _get_eapi_attrs from portage.exception import InvalidData diff --git a/lib/portage/dep/dep_check.py b/lib/portage/dep/dep_check.py index 9534590bf..625599725 100644 --- a/lib/portage/dep/dep_check.py +++ b/lib/portage/dep/dep_check.py @@ -1,8 +1,6 @@ # Copyright 2010-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - __all__ = ['dep_check', 'dep_eval', 'dep_wordreduce', 'dep_zapdeps'] import collections @@ -285,14 +283,13 @@ def dep_eval(deplist): if len(deplist) == 1: return 1 return 0 - else: - for x in deplist: - if isinstance(x, list): - if dep_eval(x)==0: - return 0 - elif x==0 or x==2: + for x in deplist: + if isinstance(x, list): + if dep_eval(x)==0: return 0 - return 1 + elif x==0 or x==2: + return 0 + return 1 class _dep_choice(SlotObject): __slots__ = ('atoms', 'slot_map', 'cp_map', 'all_available', diff --git a/lib/portage/dep/soname/SonameAtom.py b/lib/portage/dep/soname/SonameAtom.py index 5743544aa..62b2ee5e1 100644 --- a/lib/portage/dep/soname/SonameAtom.py +++ b/lib/portage/dep/soname/SonameAtom.py @@ -1,13 +1,10 @@ # Copyright 2015-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - -import sys from portage import _encodings, _unicode_encode -class SonameAtom(object): +class SonameAtom: __slots__ = ("multilib_category", "soname", "_hash_key", "_hash_value") @@ -58,14 +55,6 @@ class SonameAtom(object): def __str__(self): return "%s: %s" % (self.multilib_category, self.soname) - if sys.hexversion < 0x3000000: - - __unicode__ = __str__ - - def __str__(self): - return _unicode_encode(self.__unicode__(), - encoding=_encodings['content']) - def match(self, pkg): """ Check if the given package instance matches this atom. Unbuilt diff --git a/lib/portage/dep/soname/multilib_category.py b/lib/portage/dep/soname/multilib_category.py index 37af98705..301c62a35 100644 --- a/lib/portage/dep/soname/multilib_category.py +++ b/lib/portage/dep/soname/multilib_category.py @@ -32,8 +32,6 @@ # for given installation, we are only interested in tracking multilib # ABIs for a single OS. -from __future__ import unicode_literals - from portage.util.elf.constants import ( EF_MIPS_ABI, EF_MIPS_ABI2, diff --git a/lib/portage/dep/soname/parse.py b/lib/portage/dep/soname/parse.py index 3f3757209..d83ff12d9 100644 --- a/lib/portage/dep/soname/parse.py +++ b/lib/portage/dep/soname/parse.py @@ -1,8 +1,6 @@ # Copyright 2015 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - from portage.exception import InvalidData from portage.localization import _ from portage.dep.soname.SonameAtom import SonameAtom diff --git a/lib/portage/dispatch_conf.py b/lib/portage/dispatch_conf.py index 2fab19f1a..71693bb36 100644 --- a/lib/portage/dispatch_conf.py +++ b/lib/portage/dispatch_conf.py @@ -6,7 +6,7 @@ # Library by Wayne Davison <gentoo@blorf.net>, derived from code # written by Jeremy Wohl (http://igmus.org) -from __future__ import print_function, unicode_literals +from __future__ import print_function import errno import io @@ -97,7 +97,7 @@ def diff_mixed(func, file1, file2): if tempdir is not None: shutil.rmtree(tempdir) -class diff_mixed_wrapper(object): +class diff_mixed_wrapper: def __init__(self, f, *args): self._func = f diff --git a/lib/portage/eclass_cache.py b/lib/portage/eclass_cache.py index d2d9e2710..5a8e6ee6a 100644 --- a/lib/portage/eclass_cache.py +++ b/lib/portage/eclass_cache.py @@ -1,13 +1,10 @@ -# Copyright 2005-2014 Gentoo Foundation +# Copyright 2005-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 # Author(s): Nicholas Carpaski (carpaski@gentoo.org), Brian Harring (ferringb@gentoo.org) -from __future__ import unicode_literals - __all__ = ["cache"] import stat -import sys import operator import warnings from portage.util import normalize_path @@ -17,12 +14,8 @@ from portage import os from portage import checksum from portage import _shell_quote -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - long = int - -class hashed_path(object): +class hashed_path: def __init__(self, location): self.location = location @@ -30,11 +23,11 @@ class hashed_path(object): def __getattr__(self, attr): if attr == 'mtime': # use stat.ST_MTIME; accessing .st_mtime gets you a float - # depending on the python version, and long(float) introduces + # depending on the python version, and int(float) introduces # some rounding issues that aren't present for people using # the straight c api. # thus use the defacto python compatibility work around; - # access via index, which guarantees you get the raw long. + # access via index, which guarantees you get the raw int. try: self.mtime = obj = os.stat(self.location)[stat.ST_MTIME] except OSError as e: @@ -57,7 +50,7 @@ class hashed_path(object): def __repr__(self): return "<portage.eclass_cache.hashed_path('%s')>" % (self.location,) -class cache(object): +class cache: """ Maintains the cache information about eclasses used in ebuild. """ diff --git a/lib/portage/elog/__init__.py b/lib/portage/elog/__init__.py index cc086123f..ad6048481 100644 --- a/lib/portage/elog/__init__.py +++ b/lib/portage/elog/__init__.py @@ -3,9 +3,6 @@ # Distributed under the terms of the GNU General Public License v2 import sys -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str import portage portage.proxy.lazyimport.lazyimport(globals(), @@ -58,7 +55,7 @@ def _combine_logentries(logentries): if previous_type != msgtype: previous_type = msgtype rValue.append("%s: %s" % (msgtype, phase)) - if isinstance(msgcontent, basestring): + if isinstance(msgcontent, str): rValue.append(msgcontent.rstrip("\n")) else: for line in msgcontent: @@ -188,4 +185,3 @@ def elog_process(cpv, mysettings, phasefilter=None): noiselevel=-1) except PortageException as e: writemsg("%s\n" % str(e), noiselevel=-1) - diff --git a/lib/portage/elog/messages.py b/lib/portage/elog/messages.py index a4897d8d8..3731e6c4d 100644 --- a/lib/portage/elog/messages.py +++ b/lib/portage/elog/messages.py @@ -122,8 +122,7 @@ def _elog_base(level, msg, phase="other", key=None, color=None, out=None): if out in (sys.stdout, sys.stderr): formatted_msg = _unicode_encode(formatted_msg, encoding=_encodings['stdio'], errors='backslashreplace') - if sys.hexversion >= 0x3000000: - out = out.buffer + out = out.buffer out.write(formatted_msg) @@ -173,7 +172,7 @@ _functions = { "einfo": ("INFO", "GOOD"), "eerror": ("ERROR", "BAD"), } -class _make_msgfunction(object): +class _make_msgfunction: __slots__ = ('_color', '_level') def __init__(self, level, color): self._level = level diff --git a/lib/portage/elog/mod_echo.py b/lib/portage/elog/mod_echo.py index 8cdf6ac7d..52e8fad8d 100644 --- a/lib/portage/elog/mod_echo.py +++ b/lib/portage/elog/mod_echo.py @@ -9,9 +9,6 @@ from portage.output import EOutput, colorize from portage.const import EBUILD_PHASES from portage.localization import _ -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str _items = [] def process(mysettings, key, logentries, fulltext): @@ -61,7 +58,7 @@ def _finalize(): "ERROR": printer.eerror, "LOG": printer.einfo, "QA": printer.ewarn} - if isinstance(msgcontent, basestring): + if isinstance(msgcontent, str): msgcontent = [msgcontent] for line in msgcontent: fmap[msgtype](line.strip("\n")) diff --git a/lib/portage/elog/mod_mail_summary.py b/lib/portage/elog/mod_mail_summary.py index 0bd67f22b..31c9d25b0 100644 --- a/lib/portage/elog/mod_mail_summary.py +++ b/lib/portage/elog/mod_mail_summary.py @@ -50,7 +50,7 @@ def finalize(): def _finalize(mysettings, items): if len(items) == 0: return - elif len(items) == 1: + if len(items) == 1: count = _("one package") else: count = _("multiple packages") @@ -68,7 +68,7 @@ def _finalize(mysettings, items): mybody = _("elog messages for the following packages generated by " "process %(pid)d on host %(host)s:\n") % {"pid": os.getpid(), "host": socket.getfqdn()} for key in items: - mybody += "- %s\n" % key + mybody += "- %s\n" % key mymessage = portage.mail.create_message(myfrom, myrecipient, mysubject, mybody, attachments=list(items.values())) diff --git a/lib/portage/elog/mod_save_summary.py b/lib/portage/elog/mod_save_summary.py index e4924b66b..946a1ad4c 100644 --- a/lib/portage/elog/mod_save_summary.py +++ b/lib/portage/elog/mod_save_summary.py @@ -2,11 +2,8 @@ # Copyright 2006-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - import errno import io -import sys import time import portage from portage import os @@ -75,8 +72,6 @@ def process(mysettings, key, logentries, fulltext): mode=elogdir_grp_mode, mask=0) time_fmt = "%Y-%m-%d %H:%M:%S %Z" - if sys.hexversion < 0x3000000: - time_fmt = _unicode_encode(time_fmt) time_str = time.strftime(time_fmt, time.localtime(time.time())) # Avoid potential UnicodeDecodeError in Python 2, since strftime # returns bytes in Python 2, and %Z may contain non-ascii chars. diff --git a/lib/portage/elog/mod_syslog.py b/lib/portage/elog/mod_syslog.py index 8b26ffa1e..4dabacc52 100644 --- a/lib/portage/elog/mod_syslog.py +++ b/lib/portage/elog/mod_syslog.py @@ -2,14 +2,10 @@ # Copyright 2006-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -import sys import syslog from portage.const import EBUILD_PHASES from portage import _encodings -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str _pri = { "INFO" : syslog.LOG_INFO, @@ -25,13 +21,9 @@ def process(mysettings, key, logentries, fulltext): if not phase in logentries: continue for msgtype, msgcontent in logentries[phase]: - if isinstance(msgcontent, basestring): + if isinstance(msgcontent, str): msgcontent = [msgcontent] for line in msgcontent: line = "%s: %s: %s" % (key, phase, line) - if sys.hexversion < 0x3000000 and not isinstance(line, bytes): - # Avoid TypeError from syslog.syslog() - line = line.encode(_encodings['content'], - 'backslashreplace') syslog.syslog(_pri[msgtype], line.rstrip("\n")) syslog.closelog() diff --git a/lib/portage/emaint/main.py b/lib/portage/emaint/main.py index f448d6baa..c743f9bc7 100644 --- a/lib/portage/emaint/main.py +++ b/lib/portage/emaint/main.py @@ -13,7 +13,7 @@ from portage.module import Modules from portage.progress import ProgressBar from portage.emaint.defaults import DEFAULT_OPTIONS -class OptionItem(object): +class OptionItem: """class to hold module ArgumentParser options data """ @@ -100,7 +100,7 @@ def module_opts(module_controller, module): return _usage -class TaskHandler(object): +class TaskHandler: """Handles the running of the tasks it is given""" def __init__(self, show_progress_bar=True, verbose=True, callback=None, module_output=None): diff --git a/lib/portage/emaint/modules/binhost/binhost.py b/lib/portage/emaint/modules/binhost/binhost.py index cf9516112..1cdb3634b 100644 --- a/lib/portage/emaint/modules/binhost/binhost.py +++ b/lib/portage/emaint/modules/binhost/binhost.py @@ -1,4 +1,4 @@ -# Copyright 2005-2014 Gentoo Foundation +# Copyright 2005-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import errno @@ -9,13 +9,9 @@ from portage import os from portage.util import writemsg from portage.versions import _pkg_str -import sys -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - long = int -class BinhostHandler(object): +class BinhostHandler: short_desc = "Generate a metadata index for binary packages" @@ -54,9 +50,9 @@ class BinhostHandler(object): return False try: - if long(mtime) != s[stat.ST_MTIME]: + if int(mtime) != s[stat.ST_MTIME]: return True - if long(size) != long(s.st_size): + if int(size) != int(s.st_size): return True except ValueError: return True diff --git a/lib/portage/emaint/modules/config/config.py b/lib/portage/emaint/modules/config/config.py index a0d56992c..26edca649 100644 --- a/lib/portage/emaint/modules/config/config.py +++ b/lib/portage/emaint/modules/config/config.py @@ -6,7 +6,7 @@ from portage import os from portage.const import PRIVATE_PATH from portage.util import grabdict, writedict -class CleanConfig(object): +class CleanConfig: short_desc = "Discard any no longer installed configs from emerge's tracker list" diff --git a/lib/portage/emaint/modules/logs/logs.py b/lib/portage/emaint/modules/logs/logs.py index a3fa2b5b8..b197db718 100644 --- a/lib/portage/emaint/modules/logs/logs.py +++ b/lib/portage/emaint/modules/logs/logs.py @@ -14,7 +14,7 @@ ERROR_MESSAGES = { } -class CleanLogs(object): +class CleanLogs: short_desc = "Clean PORTAGE_LOGDIR logs" diff --git a/lib/portage/emaint/modules/merges/merges.py b/lib/portage/emaint/modules/merges/merges.py index e166150f4..a2b70edb8 100644 --- a/lib/portage/emaint/modules/merges/merges.py +++ b/lib/portage/emaint/modules/merges/merges.py @@ -11,7 +11,7 @@ import subprocess import sys import time -class TrackingFile(object): +class TrackingFile: """File for keeping track of failed merges.""" @@ -80,7 +80,7 @@ class TrackingFile(object): return self.load().items().__iter__() -class MergesHandler(object): +class MergesHandler: """Handle failed package merges.""" short_desc = "Remove failed merges" @@ -224,9 +224,9 @@ class MergesHandler(object): if output: results.append(output) if proc.returncode != os.EX_OK: - emerge_status = "Failed to emerge '%s'" % (' '.join(pkg_atoms)) + emerge_status = "Failed to emerge '%s'" % (' '.join(pkg_atoms)) else: - emerge_status = "Successfully emerged '%s'" % (' '.join(pkg_atoms)) + emerge_status = "Successfully emerged '%s'" % (' '.join(pkg_atoms)) results.append(emerge_status) return results diff --git a/lib/portage/emaint/modules/move/move.py b/lib/portage/emaint/modules/move/move.py index e9a6acb6b..e2df53fd8 100644 --- a/lib/portage/emaint/modules/move/move.py +++ b/lib/portage/emaint/modules/move/move.py @@ -7,7 +7,7 @@ from portage.exception import InvalidData from _emerge.Package import Package from portage.versions import _pkg_str -class MoveHandler(object): +class MoveHandler: def __init__(self, tree, porttree): self._tree = tree diff --git a/lib/portage/emaint/modules/resume/resume.py b/lib/portage/emaint/modules/resume/resume.py index 580643b26..759af74e9 100644 --- a/lib/portage/emaint/modules/resume/resume.py +++ b/lib/portage/emaint/modules/resume/resume.py @@ -4,7 +4,7 @@ import portage -class CleanResume(object): +class CleanResume: short_desc = "Discard emerge --resume merge lists" diff --git a/lib/portage/emaint/modules/sync/sync.py b/lib/portage/emaint/modules/sync/sync.py index ac37fdcfa..a8da68065 100644 --- a/lib/portage/emaint/modules/sync/sync.py +++ b/lib/portage/emaint/modules/sync/sync.py @@ -1,9 +1,7 @@ -# Copyright 2014-2019 Gentoo Authors +# Copyright 2014-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -import logging import os -import sys import portage portage._internal_caller = True @@ -32,13 +30,8 @@ portage.proxy.lazyimport.lazyimport(globals(), warn = create_color_func("WARN") -if sys.hexversion >= 0x3000000: - _basestring = str -else: - _basestring = basestring - -class SyncRepos(object): +class SyncRepos: short_desc = "Check repos.conf settings and/or sync repositories" @@ -125,7 +118,7 @@ class SyncRepos(object): return_messages = options.get('return-messages', False) else: return_messages = False - if isinstance(repo_names, _basestring): + if isinstance(repo_names, str): repo_names = repo_names.split() success, repos, msgs = self._get_repos(auto_sync_only=False, match_repos=repo_names) diff --git a/lib/portage/emaint/modules/world/world.py b/lib/portage/emaint/modules/world/world.py index d142c3dda..33fbaf58f 100644 --- a/lib/portage/emaint/modules/world/world.py +++ b/lib/portage/emaint/modules/world/world.py @@ -5,7 +5,7 @@ import portage from portage import os -class WorldHandler(object): +class WorldHandler: short_desc = "Fix problems in the world file" @@ -90,4 +90,3 @@ class WorldHandler(object): return (True, None) finally: world_set.unlock() - diff --git a/lib/portage/env/__init__.py b/lib/portage/env/__init__.py index 17b66d17c..1c0d15fbd 100644 --- a/lib/portage/env/__init__.py +++ b/lib/portage/env/__init__.py @@ -1,3 +1,2 @@ # Copyright: 2007 Gentoo Foundation # License: GPL2 - diff --git a/lib/portage/env/loaders.py b/lib/portage/env/loaders.py index f86988471..5fa5452af 100644 --- a/lib/portage/env/loaders.py +++ b/lib/portage/env/loaders.py @@ -69,7 +69,7 @@ def RecursiveFileLoader(filename): yield filename -class DataLoader(object): +class DataLoader: def __init__(self, validator): f = validator diff --git a/lib/portage/exception.py b/lib/portage/exception.py index a1c3c5f17..30ab0c689 100644 --- a/lib/portage/exception.py +++ b/lib/portage/exception.py @@ -1,46 +1,21 @@ -# Copyright 1998-2019 Gentoo Authors +# Copyright 1998-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import signal -import sys from portage import _encodings, _unicode_encode, _unicode_decode from portage.localization import _ -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str class PortageException(Exception): """General superclass for portage exceptions""" - if sys.hexversion >= 0x3000000: - def __init__(self, value): - self.value = value[:] - - def __str__(self): - if isinstance(self.value, str): - return self.value - else: - return repr(self.value) - else: - def __init__(self, value): - self.value = value[:] - if isinstance(self.value, basestring): - self.value = _unicode_decode(self.value, - encoding=_encodings['content'], errors='replace') - - def __unicode__(self): - if isinstance(self.value, unicode): - return self.value - else: - return _unicode_decode(repr(self.value), - encoding=_encodings['content'], errors='replace') - - def __str__(self): - if isinstance(self.value, unicode): - return _unicode_encode(self.value, - encoding=_encodings['content'], errors='backslashreplace') - else: - return repr(self.value) + def __init__(self, value): + self.value = value[:] + + def __str__(self): + if isinstance(self.value, str): + return self.value + return repr(self.value) + class PortageKeyError(KeyError, PortageException): __doc__ = KeyError.__doc__ @@ -180,7 +155,7 @@ class UnsupportedAPIException(PortagePackageException): self.cpv, self.eapi = cpv, eapi def __str__(self): eapi = self.eapi - if not isinstance(eapi, basestring): + if not isinstance(eapi, str): eapi = str(eapi) eapi = eapi.lstrip("-") msg = _("Unable to do any operations on '%(cpv)s', since " @@ -190,13 +165,6 @@ class UnsupportedAPIException(PortagePackageException): return _unicode_decode(msg, encoding=_encodings['content'], errors='replace') - if sys.hexversion < 0x3000000: - - __unicode__ = __str__ - - def __str__(self): - return _unicode_encode(self.__unicode__(), - encoding=_encodings['content'], errors='backslashreplace') class SignatureException(PortageException): """Signature was not present in the checked file""" @@ -212,4 +180,3 @@ class InvalidSignature(SignatureException): class UntrustedSignature(SignatureException): """Signature was not certified to the desired security level""" - diff --git a/lib/portage/getbinpkg.py b/lib/portage/getbinpkg.py index 997cd2eab..0bcf45b49 100644 --- a/lib/portage/getbinpkg.py +++ b/lib/portage/getbinpkg.py @@ -2,8 +2,6 @@ # Copyright 2003-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - from portage.output import colorize from portage.cache.mappings import slot_dict_class from portage.localization import _ @@ -15,6 +13,7 @@ from portage import _unicode_encode from portage.package.ebuild.fetch import _hide_url_passwd from _emerge.Package import _all_metadata_keys +import pickle import sys import socket import time @@ -25,20 +24,8 @@ import warnings _all_errors = [NotImplementedError, ValueError, socket.error] -try: - from html.parser import HTMLParser as html_parser_HTMLParser -except ImportError: - from HTMLParser import HTMLParser as html_parser_HTMLParser - -try: - from urllib.parse import unquote as urllib_parse_unquote -except ImportError: - from urllib2 import unquote as urllib_parse_unquote - -try: - import cPickle as pickle -except ImportError: - import pickle +from html.parser import HTMLParser as html_parser_HTMLParser +from urllib.parse import unquote as urllib_parse_unquote try: import ftplib @@ -48,16 +35,10 @@ else: _all_errors.extend(ftplib.all_errors) try: - try: - from http.client import HTTPConnection as http_client_HTTPConnection - from http.client import BadStatusLine as http_client_BadStatusLine - from http.client import ResponseNotReady as http_client_ResponseNotReady - from http.client import error as http_client_error - except ImportError: - from httplib import HTTPConnection as http_client_HTTPConnection - from httplib import BadStatusLine as http_client_BadStatusLine - from httplib import ResponseNotReady as http_client_ResponseNotReady - from httplib import error as http_client_error + from http.client import HTTPConnection as http_client_HTTPConnection + from http.client import BadStatusLine as http_client_BadStatusLine + from http.client import ResponseNotReady as http_client_ResponseNotReady + from http.client import error as http_client_error except ImportError as e: sys.stderr.write(colorize("BAD", "!!! CANNOT IMPORT HTTP.CLIENT: ") + str(e) + "\n") else: @@ -65,9 +46,6 @@ else: _all_errors = tuple(_all_errors) -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - long = int def make_metadata_dict(data): @@ -194,10 +172,7 @@ def create_conn(baseurl, conn=None): # http.client ImportError handler (like during stage1 -> stage2 # builds where USE=ssl is disabled for python). try: - try: - from http.client import HTTPSConnection as http_client_HTTPSConnection - except ImportError: - from httplib import HTTPSConnection as http_client_HTTPSConnection + from http.client import HTTPSConnection as http_client_HTTPSConnection except ImportError: raise NotImplementedError( _("python must have ssl enabled for https support")) @@ -460,8 +435,7 @@ def file_get_metadata(baseurl, conn=None, chunk_size=3000): if not keepconnection: conn.close() return myid - else: - xpak_data = data[len(data) - (xpaksize + 8):-8] + xpak_data = data[len(data) - (xpaksize + 8):-8] del data myid = portage.xpak.xsplit_mem(xpak_data) @@ -716,7 +690,7 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache= break # We may have metadata... now we run through the tbz2 list and check. - class CacheStats(object): + class CacheStats: from time import time def __init__(self, out): self.misses = 0 @@ -820,12 +794,11 @@ def _cmp_cpv(d1, d2): cpv2 = d2["CPV"] if cpv1 > cpv2: return 1 - elif cpv1 == cpv2: + if cpv1 == cpv2: return 0 - else: - return -1 + return -1 -class PackageIndex(object): +class PackageIndex: def __init__(self, allowed_pkg_keys=None, @@ -911,7 +884,7 @@ class PackageIndex(object): def write(self, pkgfile): if self.modified: - self.header["TIMESTAMP"] = str(long(time.time())) + self.header["TIMESTAMP"] = str(int(time.time())) self.header["PACKAGES"] = str(len(self.packages)) keys = list(self.header) keys.sort() diff --git a/lib/portage/glsa.py b/lib/portage/glsa.py index 3fd877b22..4066d6747 100644 --- a/lib/portage/glsa.py +++ b/lib/portage/glsa.py @@ -1,14 +1,11 @@ # Copyright 2003-2017 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import import io import sys -try: - from urllib.request import urlopen as urllib_request_urlopen -except ImportError: - from urllib import urlopen as urllib_request_urlopen +from urllib.request import urlopen as urllib_request_urlopen import codecs import re import operator @@ -142,7 +139,7 @@ def getListElements(listnode): if li.nodeType == xml.dom.Node.ELEMENT_NODE] return rValue -def getText(node, format, textfd = None): +def getText(node, format, textfd = None): # pylint: disable=redefined-builtin """ This is the main parser function. It takes a node and traverses recursive over the subnodes, getting the text of each (and the @@ -213,7 +210,7 @@ def getText(node, format, textfd = None): rValue = re.sub(r"[\s]{2,}", " ", rValue) return rValue -def getMultiTagsText(rootnode, tagname, format): +def getMultiTagsText(rootnode, tagname, format): # pylint: disable=redefined-builtin """ Returns a list with the text of all subnodes of type I{tagname} under I{rootnode} (which itself is not parsed) using the given I{format}. @@ -296,10 +293,9 @@ def match(atom, dbapi, match_type="default"): """ if atom[2] == "~": return revisionMatch(atom, dbapi, match_type=match_type) - elif match_type == "default" or not hasattr(dbapi, "xmatch"): + if match_type == "default" or not hasattr(dbapi, "xmatch"): return dbapi.match(atom) - else: - return dbapi.xmatch(match_type, atom) + return dbapi.xmatch(match_type, atom) def revisionMatch(revisionAtom, dbapi, match_type="default"): """ diff --git a/lib/portage/locks.py b/lib/portage/locks.py index 535698dfe..a0981712e 100644 --- a/lib/portage/locks.py +++ b/lib/portage/locks.py @@ -23,9 +23,6 @@ from portage.util import writemsg from portage.util.install_mask import _raise_exc from portage.localization import _ -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str HARDLINK_FD = -2 _HARDLINK_POLL_LATENCY = 3 # seconds @@ -87,7 +84,7 @@ def _get_lock_fn(): _open_fds = {} _open_inodes = {} -class _lock_manager(object): +class _lock_manager: __slots__ = ('fd', 'inode_key') def __init__(self, fd, fstat_result, path): self.fd = fd @@ -167,7 +164,7 @@ def _lockfile_iteration(mypath, wantnewlockfile=False, unlinkfile=False, # deprecated due to ambiguity in whether or not it's safe to close # the file descriptor, making it prone to "Bad file descriptor" errors # or file descriptor leaks. - if isinstance(mypath, basestring) and mypath[-1] == '/': + if isinstance(mypath, str) and mypath[-1] == '/': mypath = mypath[:-1] lockfilename_path = mypath @@ -192,7 +189,7 @@ def _lockfile_iteration(mypath, wantnewlockfile=False, unlinkfile=False, else: lockfilename = mypath - if isinstance(mypath, basestring): + if isinstance(mypath, str): if not os.path.exists(os.path.dirname(mypath)): raise DirectoryNotFound(os.path.dirname(mypath)) preexisting = os.path.exists(lockfilename) @@ -218,14 +215,13 @@ def _lockfile_iteration(mypath, wantnewlockfile=False, unlinkfile=False, if e.errno in (errno.ENOENT, errno.ESTALE): os.close(myfd) return None - else: - writemsg("%s: chown('%s', -1, %d)\n" % \ - (e, lockfilename, portage_gid), noiselevel=-1) - writemsg(_("Cannot chown a lockfile: '%s'\n") % \ - lockfilename, noiselevel=-1) - writemsg(_("Group IDs of current user: %s\n") % \ - " ".join(str(n) for n in os.getgroups()), - noiselevel=-1) + writemsg("%s: chown('%s', -1, %d)\n" % \ + (e, lockfilename, portage_gid), noiselevel=-1) + writemsg(_("Cannot chown a lockfile: '%s'\n") % \ + lockfilename, noiselevel=-1) + writemsg(_("Group IDs of current user: %s\n") % \ + " ".join(str(n) for n in os.getgroups()), + noiselevel=-1) finally: os.umask(old_mask) @@ -306,7 +302,7 @@ def _lockfile_iteration(mypath, wantnewlockfile=False, unlinkfile=False, os.close(myfd) lockfilename_path = _unicode_decode(lockfilename_path, encoding=_encodings['fs'], errors='strict') - if not isinstance(lockfilename_path, basestring): + if not isinstance(lockfilename_path, str): raise link_success = hardlink_lockfile(lockfilename_path, waiting_msg=waiting_msg, flags=flags) @@ -319,7 +315,7 @@ def _lockfile_iteration(mypath, wantnewlockfile=False, unlinkfile=False, raise fstat_result = None - if isinstance(lockfilename, basestring) and myfd != HARDLINK_FD and unlinkfile: + if isinstance(lockfilename, str) and myfd != HARDLINK_FD and unlinkfile: try: (removed, fstat_result) = _lockfile_was_removed(myfd, lockfilename) except Exception: @@ -333,17 +329,6 @@ def _lockfile_iteration(mypath, wantnewlockfile=False, unlinkfile=False, return None if myfd != HARDLINK_FD: - - # FD_CLOEXEC is enabled by default in Python >=3.4. - if sys.hexversion < 0x3040000: - try: - fcntl.FD_CLOEXEC - except AttributeError: - pass - else: - fcntl.fcntl(myfd, fcntl.F_SETFD, - fcntl.fcntl(myfd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) - _lock_manager(myfd, os.fstat(myfd) if fstat_result is None else fstat_result, mypath) writemsg(str((lockfilename, myfd, unlinkfile)) + "\n", 1) @@ -461,7 +446,7 @@ def unlockfile(mytuple): return True # myfd may be None here due to myfd = mypath in lockfile() - if isinstance(lockfilename, basestring) and \ + if isinstance(lockfilename, str) and \ not os.path.exists(lockfilename): writemsg(_("lockfile does not exist '%s'\n") % lockfilename, 1) if myfd is not None: @@ -474,7 +459,7 @@ def unlockfile(mytuple): unlinkfile = 1 locking_method(myfd, fcntl.LOCK_UN) except OSError: - if isinstance(lockfilename, basestring): + if isinstance(lockfilename, str): _open_fds[myfd].close() raise IOError(_("Failed to unlock file '%s'\n") % lockfilename) @@ -507,7 +492,7 @@ def unlockfile(mytuple): # why test lockfilename? because we may have been handed an # fd originally, and the caller might not like having their # open fd closed automatically on them. - if isinstance(lockfilename, basestring): + if isinstance(lockfilename, str): _open_fds[myfd].close() return True @@ -731,4 +716,3 @@ def hardlock_cleanup(path, remove_all_locks=False): pass return results - diff --git a/lib/portage/mail.py b/lib/portage/mail.py index 11923eea6..df7aac52d 100644 --- a/lib/portage/mail.py +++ b/lib/portage/mail.py @@ -1,4 +1,4 @@ -# Copyright 1998-2014 Gentoo Foundation +# Copyright 1998-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 # Since python ebuilds remove the 'email' module when USE=build @@ -19,23 +19,15 @@ from portage import _unicode_decode, _unicode_encode from portage.localization import _ import portage -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str +def _force_ascii_if_necessary(s): + # Force ascii encoding in order to avoid UnicodeEncodeError + # from smtplib.sendmail with python3 (bug #291331). + s = _unicode_encode(s, + encoding='ascii', errors='backslashreplace') + s = _unicode_decode(s, + encoding='ascii', errors='replace') + return s - def _force_ascii_if_necessary(s): - # Force ascii encoding in order to avoid UnicodeEncodeError - # from smtplib.sendmail with python3 (bug #291331). - s = _unicode_encode(s, - encoding='ascii', errors='backslashreplace') - s = _unicode_decode(s, - encoding='ascii', errors='replace') - return s - -else: - - def _force_ascii_if_necessary(s): - return s def TextMessage(_text): from email.mime.text import MIMEText @@ -50,16 +42,6 @@ def create_message(sender, recipient, subject, body, attachments=None): from email.mime.multipart import MIMEMultipart as MultipartMessage from email.utils import formatdate - if sys.hexversion < 0x3000000: - sender = _unicode_encode(sender, - encoding=_encodings['content'], errors='strict') - recipient = _unicode_encode(recipient, - encoding=_encodings['content'], errors='strict') - subject = _unicode_encode(subject, - encoding=_encodings['content'], errors='backslashreplace') - body = _unicode_encode(body, - encoding=_encodings['content'], errors='backslashreplace') - if attachments == None: mymessage = TextMessage(body) else: @@ -68,11 +50,7 @@ def create_message(sender, recipient, subject, body, attachments=None): for x in attachments: if isinstance(x, BaseMessage): mymessage.attach(x) - elif isinstance(x, basestring): - if sys.hexversion < 0x3000000: - x = _unicode_encode(x, - encoding=_encodings['content'], - errors='backslashreplace') + elif isinstance(x, str): mymessage.attach(TextMessage(x)) else: raise portage.exception.PortageException(_("Can't handle type of attachment: %s") % type(x)) @@ -132,20 +110,6 @@ def send_mail(mysettings, message): myfrom = message.get("From") - if sys.hexversion < 0x3000000: - myrecipient = _unicode_encode(myrecipient, - encoding=_encodings['content'], errors='strict') - mymailhost = _unicode_encode(mymailhost, - encoding=_encodings['content'], errors='strict') - mymailport = _unicode_encode(mymailport, - encoding=_encodings['content'], errors='strict') - myfrom = _unicode_encode(myfrom, - encoding=_encodings['content'], errors='strict') - mymailuser = _unicode_encode(mymailuser, - encoding=_encodings['content'], errors='strict') - mymailpasswd = _unicode_encode(mymailpasswd, - encoding=_encodings['content'], errors='strict') - # user wants to use a sendmail binary instead of smtp if mymailhost[0] == os.sep and os.path.exists(mymailhost): fd = os.popen(mymailhost+" -f "+myfrom+" "+myrecipient, "w") @@ -174,4 +138,3 @@ def send_mail(mysettings, message): except socket.error as e: raise portage.exception.PortageException(_("!!! A network error occurred while trying to send logmail:\n%s\nSure you configured PORTAGE_ELOG_MAILURI correctly?") % str(e)) return - diff --git a/lib/portage/manifest.py b/lib/portage/manifest.py index 4bca61e86..79c756f04 100644 --- a/lib/portage/manifest.py +++ b/lib/portage/manifest.py @@ -1,14 +1,11 @@ -# Copyright 1999-2016 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - import errno import io import logging import re import stat -import sys import warnings import portage @@ -33,12 +30,6 @@ _manifest_re = re.compile( r'^(' + '|'.join(MANIFEST2_IDENTIFIERS) + r') (\S+)( \d+( \S+ \S+)+)$', re.UNICODE) -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - _unicode = str - basestring = str -else: - _unicode = unicode class FileNotInManifestException(PortageException): pass @@ -62,21 +53,20 @@ def guessManifestFileType(filename): return None if filename.startswith("files" + os.sep): return "AUX" - elif filename.endswith(".ebuild"): + if filename.endswith(".ebuild"): return "EBUILD" - elif filename in ["ChangeLog", "metadata.xml"]: + if filename in ["ChangeLog", "metadata.xml"]: return "MISC" - else: - return "DIST" + return "DIST" def guessThinManifestFileType(filename): - type = guessManifestFileType(filename) - if type != "DIST": + filetype = guessManifestFileType(filename) + if filetype != "DIST": return None return "DIST" def parseManifest2(line): - if not isinstance(line, basestring): + if not isinstance(line, str): line = ' '.join(line) myentry = None match = _manifest_re.match(line) @@ -88,7 +78,7 @@ def parseManifest2(line): name=match.group(2), hashes=hashes) return myentry -class ManifestEntry(object): +class ManifestEntry: __slots__ = ("type", "name", "hashes") def __init__(self, **kwargs): for k, v in kwargs.items(): @@ -115,15 +105,8 @@ class Manifest2Entry(ManifestEntry): def __ne__(self, other): return not self.__eq__(other) - if sys.hexversion < 0x3000000: - - __unicode__ = __str__ - - def __str__(self): - return _unicode_encode(self.__unicode__(), - encoding=_encodings['repo.content'], errors='strict') -class Manifest(object): +class Manifest: parsers = (parseManifest2,) def __init__(self, pkgdir, distdir=None, fetchlist_dict=None, manifest1_compat=DeprecationWarning, from_scratch=False, thin=False, @@ -321,7 +304,7 @@ class Manifest(object): # thin manifests with no DIST entries, myentries is # non-empty for all currently known use cases. write_atomic(self.getFullname(), "".join("%s\n" % - _unicode(myentry) for myentry in myentries)) + str(myentry) for myentry in myentries)) self._apply_max_mtime(preserved_stats, myentries) rval = True else: diff --git a/lib/portage/metadata.py b/lib/portage/metadata.py index ffb7672d3..935bcc307 100644 --- a/lib/portage/metadata.py +++ b/lib/portage/metadata.py @@ -32,7 +32,7 @@ def action_metadata(settings, portdb, myopts, porttrees=None): auxdbkeys = portdb._known_keys - class TreeData(object): + class TreeData: __slots__ = ('dest_db', 'eclass_db', 'path', 'src_db', 'valid_nodes') def __init__(self, dest_db, eclass_db, path, src_db): self.dest_db = dest_db diff --git a/lib/portage/module.py b/lib/portage/module.py index bd7c94d4e..301bfbdba 100644 --- a/lib/portage/module.py +++ b/lib/portage/module.py @@ -19,7 +19,7 @@ class ModuleVersionError(PortageException): '''An incompatible module version''' -class Module(object): +class Module: """Class to define and hold our plug-in module @type name: string @@ -83,7 +83,7 @@ class Module(object): return mod_class -class Modules(object): +class Modules: """Dynamic modules system for loading and retrieving any of the installed emaint modules and/or provided class's diff --git a/lib/portage/news.py b/lib/portage/news.py index f7c8c7bb9..2731ea889 100644 --- a/lib/portage/news.py +++ b/lib/portage/news.py @@ -2,7 +2,7 @@ # Copyright 2006-2019 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function, unicode_literals +from __future__ import print_function __all__ = ["NewsManager", "NewsItem", "DisplayRestriction", "DisplayProfileRestriction", "DisplayKeywordRestriction", @@ -32,7 +32,7 @@ from portage.output import colorize from portage.exception import (InvalidLocation, OperationNotPermitted, PermissionDenied, ReadOnlyFileSystem) -class NewsManager(object): +class NewsManager: """ This object manages GLEP 42 style news items. It will cache news items that have previously shown up and notify users when there are relevant news @@ -200,7 +200,7 @@ _profileRE = re.compile("Display-If-Profile:(.*)\n") _keywordRE = re.compile("Display-If-Keyword:(.*)\n") _valid_profile_RE = re.compile(r'^[^*]+(/\*)?$') -class NewsItem(object): +class NewsItem: """ This class encapsulates a GLEP 42 style news item. It's purpose is to wrap parsing of these news items such that portage can determine @@ -313,7 +313,7 @@ class NewsItem(object): self._parsed = True -class DisplayRestriction(object): +class DisplayRestriction: """ A base restriction object representing a restriction of display. news items may have 'relevancy restrictions' preventing them from diff --git a/lib/portage/output.py b/lib/portage/output.py index 1070d0ef3..4c8f8a187 100644 --- a/lib/portage/output.py +++ b/lib/portage/output.py @@ -1,4 +1,4 @@ -# Copyright 1998-2015 Gentoo Foundation +# Copyright 1998-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from __future__ import division @@ -234,7 +234,7 @@ def nc_len(mystr): tmp = re.sub(esc_seq + "^m]+m", "", mystr); return len(tmp) -_legal_terms_re = re.compile(r'^(xterm|xterm-color|Eterm|aterm|rxvt|screen|kterm|rxvt-unicode|gnome|interix|tmux|st-256color)') +_legal_terms_re = re.compile(r'^(xterm|xterm-color|Eterm|aterm|rxvt|screen|kterm|rxvt-unicode|gnome|interix|tmux|st-256color|alacritty|konsole)') _disable_xtermTitle = None _max_xtermTitle_len = 253 @@ -256,9 +256,7 @@ def xtermTitle(mystr, raw=False): # avoid potential UnicodeEncodeError mystr = _unicode_encode(mystr, encoding=_encodings['stdio'], errors='backslashreplace') - f = sys.stderr - if sys.hexversion >= 0x3000000: - f = f.buffer + f = sys.stderr.buffer f.write(mystr) f.flush() @@ -336,12 +334,9 @@ def colorize(color_key, text): if havecolor: if color_key in codes: return codes[color_key] + text + codes["reset"] - elif color_key in _styles: + if color_key in _styles: return style_to_ansi_code(color_key) + text + codes["reset"] - else: - return text - else: - return text + return text compat_functions_colors = [ "bold", "white", "teal", "turquoise", "darkteal", @@ -349,7 +344,7 @@ compat_functions_colors = [ "brown", "darkyellow", "red", "darkred", ] -class create_color_func(object): +class create_color_func: __slots__ = ("_color_key",) def __init__(self, color_key): self._color_key = color_key @@ -359,7 +354,7 @@ class create_color_func(object): for c in compat_functions_colors: globals()[c] = create_color_func(c) -class ConsoleStyleFile(object): +class ConsoleStyleFile: """ A file-like object that behaves something like the colorize() function. Style identifiers @@ -398,8 +393,7 @@ class ConsoleStyleFile(object): if f in (sys.stdout, sys.stderr): s = _unicode_encode(s, encoding=_encodings['stdio'], errors='backslashreplace') - if sys.hexversion >= 0x3000000: - f = f.buffer + f = f.buffer f.write(s) def writelines(self, lines): @@ -485,7 +479,7 @@ def set_term_size(lines, columns, fd): except CommandNotFound: writemsg(_("portage: stty: command not found\n"), noiselevel=-1) -class EOutput(object): +class EOutput: """ Performs fancy terminal formatting for status and informational messages. @@ -643,7 +637,7 @@ class EOutput(object): self.__eend("ewend", errno, msg) self.__last_e_cmd = "ewend" -class ProgressBar(object): +class ProgressBar: """The interface is copied from the ProgressBar class from the EasyDialogs module (which is Mac only).""" def __init__(self, title=None, maxval=0, label=None, max_desc_length=25): @@ -755,6 +749,7 @@ class TermProgressBar(ProgressBar): bar_space = cols - percentage_str_width - square_brackets_width - 1 if self._desc: bar_space -= self._desc_max_length + if maxval == 0: max_bar_width = bar_space-3 _percent = "".ljust(percentage_str_width) @@ -779,19 +774,19 @@ class TermProgressBar(ProgressBar): "[" + (bar_width * " ") + \ "<=>" + ((max_bar_width - bar_width) * " ") + "]") return image - else: - percentage = 100 * curval // maxval - max_bar_width = bar_space - 1 - _percent = ("%d%% " % percentage).rjust(percentage_str_width) - image = "%s%s" % (self._desc, _percent) - if cols < min_columns: - return image - offset = curval / maxval - bar_width = int(offset * max_bar_width) - image = image + "[" + (bar_width * "=") + \ - ">" + ((max_bar_width - bar_width) * " ") + "]" + percentage = 100 * curval // maxval + max_bar_width = bar_space - 1 + _percent = ("%d%% " % percentage).rjust(percentage_str_width) + image = "%s%s" % (self._desc, _percent) + + if cols < min_columns: return image + offset = curval / maxval + bar_width = int(offset * max_bar_width) + image = image + "[" + (bar_width * "=") + \ + ">" + ((max_bar_width - bar_width) * " ") + "]" + return image _color_map_loaded = False diff --git a/lib/portage/package/ebuild/_config/KeywordsManager.py b/lib/portage/package/ebuild/_config/KeywordsManager.py index 48397b022..136f57c35 100644 --- a/lib/portage/package/ebuild/_config/KeywordsManager.py +++ b/lib/portage/package/ebuild/_config/KeywordsManager.py @@ -16,7 +16,7 @@ from portage.package.ebuild._config.helper import ordered_by_atom_specificity from portage.util import grabdict_package, stack_lists, writemsg from portage.versions import _pkg_str -class KeywordsManager(object): +class KeywordsManager: """Manager class to handle keywords processing and validation""" def __init__(self, profiles, abs_user_config, user_config=True, diff --git a/lib/portage/package/ebuild/_config/LicenseManager.py b/lib/portage/package/ebuild/_config/LicenseManager.py index 1d4e08207..fcffdb12e 100644 --- a/lib/portage/package/ebuild/_config/LicenseManager.py +++ b/lib/portage/package/ebuild/_config/LicenseManager.py @@ -15,7 +15,7 @@ from portage.versions import cpv_getkey, _pkg_str from portage.package.ebuild._config.helper import ordered_by_atom_specificity -class LicenseManager(object): +class LicenseManager: def __init__(self, license_group_locations, abs_user_config, user_config=True): diff --git a/lib/portage/package/ebuild/_config/LocationsManager.py b/lib/portage/package/ebuild/_config/LocationsManager.py index 75320258f..b90b9227c 100644 --- a/lib/portage/package/ebuild/_config/LocationsManager.py +++ b/lib/portage/package/ebuild/_config/LocationsManager.py @@ -1,8 +1,6 @@ # Copyright 2010-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - __all__ = ( 'LocationsManager', ) @@ -37,7 +35,7 @@ _profile_node = collections.namedtuple('_profile_node', _allow_parent_colon = frozenset( ["portage-2"]) -class LocationsManager(object): +class LocationsManager: def __init__(self, config_root=None, eprefix=None, config_profile_path=None, local_config=True, \ target_root=None, sysroot=None): diff --git a/lib/portage/package/ebuild/_config/MaskManager.py b/lib/portage/package/ebuild/_config/MaskManager.py index 40cc6e0c4..7714456e1 100644 --- a/lib/portage/package/ebuild/_config/MaskManager.py +++ b/lib/portage/package/ebuild/_config/MaskManager.py @@ -13,7 +13,7 @@ from portage.localization import _ from portage.util import append_repo, grabfile_package, stack_lists, writemsg from portage.versions import _pkg_str -class MaskManager(object): +class MaskManager: def __init__(self, repositories, profiles, abs_user_config, user_config=True, strict_umatched_removal=False): diff --git a/lib/portage/package/ebuild/_config/UseManager.py b/lib/portage/package/ebuild/_config/UseManager.py index 7302876ab..190ab1b7c 100644 --- a/lib/portage/package/ebuild/_config/UseManager.py +++ b/lib/portage/package/ebuild/_config/UseManager.py @@ -16,7 +16,7 @@ from portage.versions import _pkg_str from portage.package.ebuild._config.helper import ordered_by_atom_specificity -class UseManager(object): +class UseManager: def __init__(self, repositories, profiles, abs_user_config, is_stable, user_config=True): diff --git a/lib/portage/package/ebuild/_config/VirtualsManager.py b/lib/portage/package/ebuild/_config/VirtualsManager.py index c4d1e3635..a08c969ac 100644 --- a/lib/portage/package/ebuild/_config/VirtualsManager.py +++ b/lib/portage/package/ebuild/_config/VirtualsManager.py @@ -14,7 +14,7 @@ from portage.localization import _ from portage.util import grabdict, stack_dictlist, writemsg from portage.versions import cpv_getkey -class VirtualsManager(object): +class VirtualsManager: def __init__(self, *args, **kwargs): if kwargs.get("_copy"): diff --git a/lib/portage/package/ebuild/_config/features_set.py b/lib/portage/package/ebuild/_config/features_set.py index 62236fd89..d38874379 100644 --- a/lib/portage/package/ebuild/_config/features_set.py +++ b/lib/portage/package/ebuild/_config/features_set.py @@ -12,7 +12,7 @@ from portage.localization import _ from portage.output import colorize from portage.util import writemsg_level -class features_set(object): +class features_set: """ Provides relevant set operations needed for access and modification of config.features. The FEATURES variable is automatically synchronized diff --git a/lib/portage/package/ebuild/_config/special_env_vars.py b/lib/portage/package/ebuild/_config/special_env_vars.py index 12d701c9a..8a9161c1b 100644 --- a/lib/portage/package/ebuild/_config/special_env_vars.py +++ b/lib/portage/package/ebuild/_config/special_env_vars.py @@ -1,8 +1,6 @@ -# Copyright 2010-2019 Gentoo Authors +# Copyright 2010-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - __all__ = ( 'case_insensitive_vars', 'default_globals', 'env_blacklist', \ 'environ_filter', 'environ_whitelist', 'environ_whitelist_re', @@ -184,7 +182,7 @@ environ_filter += [ "PORTAGE_RO_DISTDIRS", "PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS", "PORTAGE_RSYNC_RETRIES", "PORTAGE_SSH_OPTS", "PORTAGE_SYNC_STALE", - "PORTAGE_USE", + "PORTAGE_USE", "PORTAGE_LOG_FILTER_FILE_CMD", "PORTAGE_LOGDIR", "PORTAGE_LOGDIR_CLEAN", "QUICKPKG_DEFAULT_OPTS", "REPOMAN_DEFAULT_OPTS", "RESUMECOMMAND", "RESUMECOMMAND_FTP", @@ -213,7 +211,9 @@ default_globals = { 'PORTAGE_BZIP2_COMMAND': 'bzip2', } -validate_commands = ('PORTAGE_BZIP2_COMMAND', 'PORTAGE_BUNZIP2_COMMAND',) +validate_commands = ('PORTAGE_BZIP2_COMMAND', 'PORTAGE_BUNZIP2_COMMAND', + 'PORTAGE_LOG_FILTER_FILE_CMD', +) # To enhance usability, make some vars case insensitive # by forcing them to lower case. diff --git a/lib/portage/package/ebuild/_ipc/IpcCommand.py b/lib/portage/package/ebuild/_ipc/IpcCommand.py index efb27f0a2..42a612753 100644 --- a/lib/portage/package/ebuild/_ipc/IpcCommand.py +++ b/lib/portage/package/ebuild/_ipc/IpcCommand.py @@ -1,7 +1,7 @@ # Copyright 2010 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -class IpcCommand(object): +class IpcCommand: __slots__ = () diff --git a/lib/portage/package/ebuild/_ipc/QueryCommand.py b/lib/portage/package/ebuild/_ipc/QueryCommand.py index fa6d1ea16..7af465234 100644 --- a/lib/portage/package/ebuild/_ipc/QueryCommand.py +++ b/lib/portage/package/ebuild/_ipc/QueryCommand.py @@ -1,8 +1,6 @@ # Copyright 2010-2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - import io import portage @@ -84,10 +82,10 @@ class QueryCommand(IpcCommand): else: returncode = 1 return ('', warnings_str, returncode) - elif cmd == 'best_version': + if cmd == 'best_version': m = best(vardb.match(atom)) return ('%s\n' % m, warnings_str, 0) - elif cmd in ('master_repositories', 'repository_path', 'available_eclasses', 'eclass_path', 'license_path'): + if cmd in ('master_repositories', 'repository_path', 'available_eclasses', 'eclass_path', 'license_path'): repo = _repo_name_re.match(args[0]) if repo is None: return ('', '%s: Invalid repository: %s\n' % (cmd, args[0]), 2) @@ -98,24 +96,23 @@ class QueryCommand(IpcCommand): if cmd == 'master_repositories': return ('%s\n' % ' '.join(x.name for x in repo.masters), warnings_str, 0) - elif cmd == 'repository_path': + if cmd == 'repository_path': return ('%s\n' % repo.location, warnings_str, 0) - elif cmd == 'available_eclasses': + if cmd == 'available_eclasses': return ('%s\n' % ' '.join(sorted(repo.eclass_db.eclasses)), warnings_str, 0) - elif cmd == 'eclass_path': + if cmd == 'eclass_path': try: eclass = repo.eclass_db.eclasses[args[1]] except KeyError: return ('', warnings_str, 1) return ('%s\n' % eclass.location, warnings_str, 0) - elif cmd == 'license_path': + if cmd == 'license_path': paths = reversed([os.path.join(x.location, 'licenses', args[1]) for x in list(repo.masters) + [repo]]) for path in paths: if os.path.exists(path): return ('%s\n' % path, warnings_str, 0) return ('', warnings_str, 1) - else: - return ('', 'Invalid command: %s\n' % cmd, 3) + return ('', 'Invalid command: %s\n' % cmd, 3) def _elog(self, elog_funcname, lines): """ diff --git a/lib/portage/package/ebuild/_parallel_manifest/ManifestProcess.py b/lib/portage/package/ebuild/_parallel_manifest/ManifestProcess.py index 44e257664..cf718a796 100644 --- a/lib/portage/package/ebuild/_parallel_manifest/ManifestProcess.py +++ b/lib/portage/package/ebuild/_parallel_manifest/ManifestProcess.py @@ -39,5 +39,4 @@ class ManifestProcess(ForkProcess): else: if modified: return self.MODIFIED - else: - return os.EX_OK + return os.EX_OK diff --git a/lib/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py b/lib/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py index fabea9bc1..2ac736bc6 100644 --- a/lib/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py +++ b/lib/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py @@ -84,5 +84,3 @@ class ManifestScheduler(AsyncScheduler): noiselevel=-1) AsyncScheduler._task_exit(self, task) - - diff --git a/lib/portage/package/ebuild/config.py b/lib/portage/package/ebuild/config.py index c76474354..b1b2b47d7 100644 --- a/lib/portage/package/ebuild/config.py +++ b/lib/portage/package/ebuild/config.py @@ -1,8 +1,6 @@ -# Copyright 2010-2019 Gentoo Authors +# Copyright 2010-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - __all__ = [ 'autouse', 'best_from_dict', 'check_config_instance', 'config', ] @@ -68,9 +66,6 @@ from portage.package.ebuild._config.VirtualsManager import VirtualsManager from portage.package.ebuild._config.helper import ordered_by_atom_specificity, prune_incremental from portage.package.ebuild._config.unpack_dependencies import load_unpack_dependencies_configuration -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str _feature_flags_cache = {} @@ -102,12 +97,10 @@ def best_from_dict(key, top_dict, key_order, EmptyOnError=1, FullCopy=1, AllowEm if x in top_dict and key in top_dict[x]: if FullCopy: return copy.deepcopy(top_dict[x][key]) - else: - return top_dict[x][key] + return top_dict[x][key] if EmptyOnError: return "" - else: - raise KeyError("Key not found in list; '%s'" % key) + raise KeyError("Key not found in list; '%s'" % key) def _lazy_iuse_regex(iuse_implicit): """ @@ -122,7 +115,7 @@ def _lazy_iuse_regex(iuse_implicit): regex = regex.replace("\\.\\*", ".*") return regex -class _iuse_implicit_match_cache(object): +class _iuse_implicit_match_cache: def __init__(self, settings): self._iuse_implicit_re = re.compile("^(%s)$" % \ @@ -140,7 +133,7 @@ class _iuse_implicit_match_cache(object): self._cache[flag] = m return m -class config(object): +class config: """ This class encompasses the main portage configuration. Data is pulled from ROOT/PORTDIR/profiles/, from ROOT/etc/make.profile incrementally through all @@ -1324,7 +1317,7 @@ class config(object): self.useforce = self._use_manager.getUseForce() self.regenerate() - class _lazy_vars(object): + class _lazy_vars: __slots__ = ('built_use', 'settings', 'values') @@ -1357,7 +1350,7 @@ class config(object): restrict = set() return ' '.join(sorted(restrict)) - class _lazy_use_expand(object): + class _lazy_use_expand: """ Lazily evaluate USE_EXPAND variables since they are only needed when an ebuild shell is spawned. Variables values are made consistent with @@ -1456,7 +1449,7 @@ class config(object): pkg = None built_use = None explicit_iuse = None - if not isinstance(mycpv, basestring): + if not isinstance(mycpv, str): pkg = mycpv mycpv = pkg.cpv mydb = pkg._metadata @@ -2645,10 +2638,10 @@ class config(object): # portage plans to update itself. if mykey == "PORTAGE_BIN_PATH": return portage._bin_path - elif mykey == "PORTAGE_PYM_PATH": + if mykey == "PORTAGE_PYM_PATH": return portage._pym_path - elif mykey == "PORTAGE_PYTHONPATH": + if mykey == "PORTAGE_PYTHONPATH": value = [x for x in \ self.backupenv.get("PYTHONPATH", "").split(":") if x] need_pym_path = True @@ -2662,7 +2655,7 @@ class config(object): value.insert(0, portage._pym_path) return ":".join(value) - elif mykey == "PORTAGE_GID": + if mykey == "PORTAGE_GID": return "%s" % portage_gid for d in self.lookuplist: @@ -2705,7 +2698,7 @@ class config(object): def __contains__(self, mykey): """Called to implement membership test operators (in and not in).""" try: - self._getitem(mykey) + self._getitem(mykey) except KeyError: return False else: @@ -2715,12 +2708,8 @@ class config(object): v = self.get(k) if v is not None: return v - else: - self[k] = x - return x - - def keys(self): - return list(self) + self[k] = x + return x def __iter__(self): keys = set() @@ -2736,12 +2725,9 @@ class config(object): for k in self: yield (k, self._getitem(k)) - def items(self): - return list(self.iteritems()) - def __setitem__(self,mykey,myvalue): "set a value; will be thrown away at reset() time" - if not isinstance(myvalue, basestring): + if not isinstance(myvalue, str): raise ValueError("Invalid type being used as a value: '%s': '%s'" % (str(mykey),str(myvalue))) # Avoid potential UnicodeDecodeError exceptions later. @@ -2774,7 +2760,7 @@ class config(object): for x, myvalue in self.iteritems(): if x in environ_filter: continue - if not isinstance(myvalue, basestring): + if not isinstance(myvalue, str): writemsg(_("!!! Non-string value in config: %s=%s\n") % \ (x, myvalue), noiselevel=-1) continue @@ -2929,6 +2915,5 @@ class config(object): return self._selinux_enabled - if sys.hexversion >= 0x3000000: - keys = __iter__ - items = iteritems + keys = __iter__ + items = iteritems diff --git a/lib/portage/package/ebuild/doebuild.py b/lib/portage/package/ebuild/doebuild.py index 7d685dbe4..1a667a03d 100644 --- a/lib/portage/package/ebuild/doebuild.py +++ b/lib/portage/package/ebuild/doebuild.py @@ -1,8 +1,6 @@ # Copyright 2010-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - __all__ = ['doebuild', 'doebuild_environment', 'spawn', 'spawnebuild'] import grp @@ -95,10 +93,6 @@ from _emerge.EbuildSpawnProcess import EbuildSpawnProcess from _emerge.Package import Package from _emerge.RootConfig import RootConfig -if sys.hexversion >= 0x3000000: - _unicode = str -else: - _unicode = unicode _unsandboxed_phases = frozenset([ "clean", "cleanrm", "config", @@ -851,7 +845,7 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0, if returnpid: return _spawn_phase(mydo, mysettings, fd_pipes=fd_pipes, returnpid=returnpid) - elif dbkey and dbkey is not DeprecationWarning: + if dbkey and dbkey is not DeprecationWarning: mysettings["dbkey"] = dbkey else: mysettings["dbkey"] = \ @@ -860,7 +854,7 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0, return _spawn_phase(mydo, mysettings, fd_pipes=fd_pipes, returnpid=returnpid) - elif mydo == "nofetch": + if mydo == "nofetch": if returnpid: writemsg("!!! doebuild: %s\n" % @@ -1140,11 +1134,11 @@ def doebuild(myebuild, mydo, _unused=DeprecationWarning, settings=None, debug=0, mf = None _doebuild_manifest_cache = None return not digestgen(mysettings=mysettings, myportdb=mydbapi) - elif mydo == "digest": + if mydo == "digest": mf = None _doebuild_manifest_cache = None return not digestgen(mysettings=mysettings, myportdb=mydbapi) - elif "digest" in mysettings.features: + if "digest" in mysettings.features: mf = None _doebuild_manifest_cache = None digestgen(mysettings=mysettings, myportdb=mydbapi) @@ -1430,7 +1424,7 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi): mydbapi.aux_get(mysettings.mycpv, all_keys, myrepo=mysettings.get("PORTAGE_REPO_NAME")))) - class FakeTree(object): + class FakeTree: def __init__(self, mydb): self.dbapi = mydb @@ -2489,7 +2483,7 @@ def _post_src_install_soname_symlinks(mysettings, out): # Compute the multilib category and write it back to the file. entry.multilib_category = compute_multilib_category(elf_header) - needed_file.write(_unicode(entry)) + needed_file.write(str(entry)) if entry.multilib_category is None: if not qa_prebuilt or qa_prebuilt.match( @@ -2540,7 +2534,7 @@ def _post_src_install_soname_symlinks(mysettings, out): if unrecognized_elf_files: qa_msg = ["QA Notice: Unrecognized ELF file(s):"] qa_msg.append("") - qa_msg.extend("\t%s" % _unicode(entry).rstrip() + qa_msg.extend("\t%s" % str(entry).rstrip() for entry in unrecognized_elf_files) qa_msg.append("") for line in qa_msg: diff --git a/lib/portage/package/ebuild/fetch.py b/lib/portage/package/ebuild/fetch.py index 11b13fe56..5a55c4076 100644 --- a/lib/portage/package/ebuild/fetch.py +++ b/lib/portage/package/ebuild/fetch.py @@ -20,16 +20,8 @@ import tempfile import time from collections import OrderedDict - -try: - from urllib.parse import urlparse -except ImportError: - from urlparse import urlparse - -try: - from urllib.parse import quote as urlquote -except ImportError: - from urllib import quote as urlquote +from urllib.parse import urlparse +from urllib.parse import quote as urlquote import portage portage.proxy.lazyimport.lazyimport(globals(), @@ -355,7 +347,7 @@ _size_suffix_map = { } -class FlatLayout(object): +class FlatLayout: def get_path(self, filename): return filename @@ -375,7 +367,7 @@ class FlatLayout(object): return len(args) == 1 -class FilenameHashLayout(object): +class FilenameHashLayout: def __init__(self, algo, cutoffs): self.algo = algo self.cutoffs = [int(x) for x in cutoffs.split(':')] @@ -424,7 +416,7 @@ class FilenameHashLayout(object): return False -class MirrorLayoutConfig(object): +class MirrorLayoutConfig: """ Class to read layout.conf from a mirror. """ @@ -462,11 +454,10 @@ class MirrorLayoutConfig(object): if self.validate_structure(val): if val[0] == 'flat': return FlatLayout(*val[1:]) - elif val[0] == 'filename-hash': + if val[0] == 'filename-hash': return FilenameHashLayout(*val[1:]) - else: - # fallback - return FlatLayout() + # fallback + return FlatLayout() def get_all_layouts(self): ret = [] @@ -1258,8 +1249,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, writemsg(_("!!! File %s is incorrect size, " "but unable to retry.\n") % myfile, noiselevel=-1) return 0 - else: - continue + continue if fetched != 2 and has_space: #we either need to resume or start the download diff --git a/lib/portage/package/ebuild/getmaskingreason.py b/lib/portage/package/ebuild/getmaskingreason.py index 1e4ed21ce..22232de29 100644 --- a/lib/portage/package/ebuild/getmaskingreason.py +++ b/lib/portage/package/ebuild/getmaskingreason.py @@ -54,8 +54,7 @@ def getmaskingreason(mycpv, metadata=None, settings=None, # contain essential things like SLOT. if return_location: return (None, None) - else: - return None + return None # Sometimes we can't access SLOT or repository due to corruption. pkg = mycpv @@ -114,13 +113,11 @@ def getmaskingreason(mycpv, metadata=None, settings=None, comment = "" if return_location: return (comment, pmask_filename) - else: - return comment + return comment elif comment_valid != -1: # Apparently this comment applies to multiple masks, so # it remains valid until a blank line is encountered. comment_valid += 1 if return_location: return (None, None) - else: - return None + return None diff --git a/lib/portage/package/ebuild/getmaskingstatus.py b/lib/portage/package/ebuild/getmaskingstatus.py index 4b9e588f7..cf2b7344b 100644 --- a/lib/portage/package/ebuild/getmaskingstatus.py +++ b/lib/portage/package/ebuild/getmaskingstatus.py @@ -1,11 +1,8 @@ -# Copyright 2010-2014 Gentoo Foundation +# Copyright 2010-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - __all__ = ['getmaskingstatus'] -import sys import portage from portage import eapi_is_supported, _eapi_is_deprecated @@ -14,11 +11,8 @@ from portage.localization import _ from portage.package.ebuild.config import config from portage.versions import catpkgsplit, _pkg_str -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str -class _UnmaskHint(object): +class _UnmaskHint: __slots__ = ('key', 'value') @@ -26,7 +20,7 @@ class _UnmaskHint(object): self.key = key self.value = value -class _MaskReason(object): +class _MaskReason: __slots__ = ('category', 'message', 'unmask_hint') @@ -48,7 +42,7 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): metadata = None installed = False - if not isinstance(mycpv, basestring): + if not isinstance(mycpv, str): # emerge passed in a Package instance pkg = mycpv mycpv = pkg.cpv @@ -91,7 +85,7 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): restrict = metadata["RESTRICT"] if not eapi_is_supported(eapi): return [_MaskReason("EAPI", "EAPI %s" % eapi)] - elif _eapi_is_deprecated(eapi) and not installed: + if _eapi_is_deprecated(eapi) and not installed: return [_MaskReason("EAPI", "EAPI %s" % eapi)] egroups = settings.configdict["backupenv"].get( "ACCEPT_KEYWORDS", "").split() diff --git a/lib/portage/package/ebuild/prepare_build_dirs.py b/lib/portage/package/ebuild/prepare_build_dirs.py index 8349d306f..b63cd89fa 100644 --- a/lib/portage/package/ebuild/prepare_build_dirs.py +++ b/lib/portage/package/ebuild/prepare_build_dirs.py @@ -1,8 +1,6 @@ # Copyright 2010-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - __all__ = ['prepare_build_dirs'] import errno diff --git a/lib/portage/process.py b/lib/portage/process.py index 3e5d2e4ef..57f6e15a2 100644 --- a/lib/portage/process.py +++ b/lib/portage/process.py @@ -1,5 +1,5 @@ # portage.py -- core Portage functionality -# Copyright 1998-2019 Gentoo Authors +# Copyright 1998-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 @@ -40,9 +40,6 @@ try: except ImportError: max_fd_limit = 256 -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str # Support PEP 446 for Python >=3.4 try: @@ -112,7 +109,7 @@ def sanitize_fds(): if _set_inheritable is not None: whitelist = frozenset([ - sys.__stdin__.fileno(), + portage._get_stdin().fileno(), sys.__stdout__.fileno(), sys.__stderr__.fileno(), ]) @@ -216,10 +213,7 @@ def run_exitfuncs(): exc_info = sys.exc_info() if exc_info is not None: - if sys.hexversion >= 0x3000000: - raise exc_info[0](exc_info[1]).with_traceback(exc_info[2]) - else: - exec("raise exc_info[0], exc_info[1], exc_info[2]") + raise exc_info[0](exc_info[1]).with_traceback(exc_info[2]) atexit.register(run_exitfuncs) @@ -244,7 +238,7 @@ def cleanup(): def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False, uid=None, gid=None, groups=None, umask=None, cwd=None, logfile=None, path_lookup=True, pre_exec=None, - close_fds=(sys.version_info < (3, 4)), unshare_net=False, + close_fds=False, unshare_net=False, unshare_ipc=False, unshare_mount=False, unshare_pid=False, cgroup=None): """ @@ -302,20 +296,11 @@ def spawn(mycommand, env=None, opt_name=None, fd_pipes=None, returnpid=False, """ # mycommand is either a str or a list - if isinstance(mycommand, basestring): + if isinstance(mycommand, str): mycommand = mycommand.split() env = os.environ if env is None else env - if sys.hexversion < 0x3000000: - # Avoid a potential UnicodeEncodeError from os.execve(). - env_bytes = {} - for k, v in env.items(): - env_bytes[_unicode_encode(k, encoding=_encodings['content'])] = \ - _unicode_encode(v, encoding=_encodings['content']) - env = env_bytes - del env_bytes - # If an absolute path to an executable file isn't given # search for it unless we've been told not to. binary = mycommand[0] @@ -767,7 +752,7 @@ def _exec(binary, mycommand, opt_name, fd_pipes, os.execve(binary, myargs, env) -class _unshare_validator(object): +class _unshare_validator: """ In order to prevent failed unshare calls from corrupting the state of an essential process, validate the relevant unshare call in a @@ -982,7 +967,7 @@ def find_binary(binary): @return: full path to binary or None if the binary could not be located. """ paths = os.environ.get("PATH", "") - if sys.hexversion >= 0x3000000 and isinstance(binary, bytes): + if isinstance(binary, bytes): # return bytes when input is bytes paths = paths.encode(sys.getfilesystemencoding(), 'surrogateescape') paths = paths.split(b':') diff --git a/lib/portage/progress.py b/lib/portage/progress.py index e43c2afbd..0be790106 100644 --- a/lib/portage/progress.py +++ b/lib/portage/progress.py @@ -7,7 +7,7 @@ import signal import portage -class ProgressHandler(object): +class ProgressHandler: def __init__(self): self.reset() @@ -58,4 +58,3 @@ class ProgressBar(ProgressHandler): def stop(self): signal.signal(signal.SIGWINCH, signal.SIG_DFL) - diff --git a/lib/portage/proxy/lazyimport.py b/lib/portage/proxy/lazyimport.py index d4258706d..532f3ce0a 100644 --- a/lib/portage/proxy/lazyimport.py +++ b/lib/portage/proxy/lazyimport.py @@ -1,4 +1,4 @@ -# Copyright 2009-2014 Gentoo Foundation +# Copyright 2009-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 __all__ = ['lazyimport'] @@ -13,9 +13,6 @@ except ImportError: from portage.proxy.objectproxy import ObjectProxy -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str _module_proxies = {} _module_proxies_lock = threading.RLock() @@ -169,7 +166,7 @@ def lazyimport(scope, *args): if len(parts) == 1: name = s - if not name or not isinstance(name, basestring): + if not name or not isinstance(name, str): raise ValueError(name) components = name.split('.') diff --git a/lib/portage/proxy/objectproxy.py b/lib/portage/proxy/objectproxy.py index a755774ae..7d03dbef6 100644 --- a/lib/portage/proxy/objectproxy.py +++ b/lib/portage/proxy/objectproxy.py @@ -1,11 +1,10 @@ -# Copyright 2008-2012 Gentoo Foundation +# Copyright 2008-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -import sys __all__ = ['ObjectProxy'] -class ObjectProxy(object): +class ObjectProxy: """ Object that acts as a proxy to another object, forwarding @@ -88,11 +87,5 @@ class ObjectProxy(object): def __bool__(self): return bool(object.__getattribute__(self, '_get_target')()) - if sys.hexversion < 0x3000000: - __nonzero__ = __bool__ - - def __unicode__(self): - return unicode(object.__getattribute__(self, '_get_target')()) - def __int__(self): return int(object.__getattribute__(self, '_get_target')()) diff --git a/lib/portage/repository/config.py b/lib/portage/repository/config.py index 50ab18026..0f3e582f8 100644 --- a/lib/portage/repository/config.py +++ b/lib/portage/repository/config.py @@ -1,12 +1,9 @@ -# Copyright 2010-2019 Gentoo Authors +# Copyright 2010-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - import io import logging import warnings -import sys import re import portage @@ -28,9 +25,6 @@ from portage import _encodings from portage import manifest import portage.sync -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str # Characters prohibited by repoman's file.name check. _invalid_path_char_re = re.compile(r'[^a-zA-Z0-9._\-+/]') @@ -71,7 +65,7 @@ def _find_invalid_path_char(path, pos=0, endpos=None): return -1 -class RepoConfig(object): +class RepoConfig: """Stores config of one repository""" __slots__ = ( @@ -113,6 +107,7 @@ class RepoConfig(object): 'sync_hooks_only_on_change', 'sync_openpgp_keyserver', 'sync_openpgp_key_path', + 'sync_openpgp_key_refresh', 'sync_openpgp_key_refresh_retry_count', 'sync_openpgp_key_refresh_retry_delay_exp_base', 'sync_openpgp_key_refresh_retry_delay_max', @@ -219,10 +214,10 @@ class RepoConfig(object): self.sync_depth = repo_opts.get('sync-depth') self.sync_hooks_only_on_change = repo_opts.get( - 'sync-hooks-only-on-change', 'false').lower() == 'true' + 'sync-hooks-only-on-change', 'false').lower() in ('true', 'yes') self.strict_misc_digests = repo_opts.get( - 'strict-misc-digests', 'true').lower() == 'true' + 'strict-misc-digests', 'true').lower() in ('true', 'yes') self.sync_allow_hardlinks = repo_opts.get( 'sync-allow-hardlinks', 'true').lower() in ('true', 'yes') @@ -233,6 +228,9 @@ class RepoConfig(object): self.sync_openpgp_key_path = repo_opts.get( 'sync-openpgp-key-path', None) + self.sync_openpgp_key_refresh = repo_opts.get( + 'sync-openpgp-key-refresh', 'true').lower() in ('true', 'yes') + for k in ('sync_openpgp_key_refresh_retry_count', 'sync_openpgp_key_refresh_retry_delay_exp_base', 'sync_openpgp_key_refresh_retry_delay_max', @@ -259,10 +257,10 @@ class RepoConfig(object): self.module_specific_options = {} # Not implemented. - format = repo_opts.get('format') - if format is not None: - format = format.strip() - self.format = format + repo_format = repo_opts.get('format') + if repo_format is not None: + repo_format = repo_format.strip() + self.format = repo_format self.user_location = None location = repo_opts.get('location') @@ -497,6 +495,8 @@ class RepoConfig(object): repo_msg.append(indent + "location: " + self.location) if not self.strict_misc_digests: repo_msg.append(indent + "strict-misc-digests: false") + if not self.sync_openpgp_key_refresh: + repo_msg.append(indent + "sync-openpgp-key-refresh: no") if self.sync_type: repo_msg.append(indent + "sync-type: " + self.sync_type) if self.sync_umask: @@ -529,14 +529,8 @@ class RepoConfig(object): d[k] = getattr(self, k, None) return "%s" % (d,) - if sys.hexversion < 0x3000000: - - __unicode__ = __str__ - def __str__(self): - return _unicode_encode(self.__unicode__()) - -class RepoConfigLoader(object): +class RepoConfigLoader: """Loads and store config of several repositories, loaded from PORTDIR_OVERLAY or repos.conf""" @staticmethod @@ -609,6 +603,7 @@ class RepoConfigLoader(object): 'sync_hooks_only_on_change', 'sync_openpgp_keyserver', 'sync_openpgp_key_path', + 'sync_openpgp_key_refresh', 'sync_openpgp_key_refresh_retry_count', 'sync_openpgp_key_refresh_retry_delay_exp_base', 'sync_openpgp_key_refresh_retry_delay_max', @@ -664,7 +659,7 @@ class RepoConfigLoader(object): recursive_paths = [] for p in paths: - if isinstance(p, basestring): + if isinstance(p, str): recursive_paths.extend(_recursive_file_list(p)) else: recursive_paths.append(p) @@ -973,8 +968,7 @@ class RepoConfigLoader(object): main_repo = self.prepos['DEFAULT'].main_repo if main_repo is not None and main_repo in self.prepos: return self.prepos[main_repo].location - else: - return '' + return '' def mainRepo(self): """Returns the main repo""" @@ -1047,6 +1041,7 @@ class RepoConfigLoader(object): bool_keys = ( "strict_misc_digests", "sync_allow_hardlinks", + "sync_openpgp_key_refresh", "sync_rcu", ) str_or_int_keys = ( diff --git a/lib/portage/repository/storage/hardlink_rcu.py b/lib/portage/repository/storage/hardlink_rcu.py index 80cdbb0d7..bb2c8496b 100644 --- a/lib/portage/repository/storage/hardlink_rcu.py +++ b/lib/portage/repository/storage/hardlink_rcu.py @@ -1,4 +1,4 @@ -# Copyright 2018 Gentoo Foundation +# Copyright 2018-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import datetime @@ -204,6 +204,17 @@ class HardlinkRcuRepoStorage(RepoStorageInterface): except OSError: pass os.symlink('snapshots/{}'.format(new_id), new_symlink) + + # If SyncManager.pre_sync creates an empty directory where + # self._latest_symlink is suppose to be (which is normal if + # sync-rcu-store-dir has been removed), then we need to remove + # the directory or else rename will raise IsADirectoryError + # when we try to replace the directory with a symlink. + try: + os.rmdir(self._latest_symlink) + except OSError: + pass + os.rename(new_symlink, self._latest_symlink) try: diff --git a/lib/portage/repository/storage/interface.py b/lib/portage/repository/storage/interface.py index f83c42b84..ce8a2a170 100644 --- a/lib/portage/repository/storage/interface.py +++ b/lib/portage/repository/storage/interface.py @@ -11,7 +11,7 @@ class RepoStorageException(PortageException): """ -class RepoStorageInterface(object): +class RepoStorageInterface: """ Abstract repository storage interface. diff --git a/lib/portage/sync/config_checks.py b/lib/portage/sync/config_checks.py index db316aa88..3157d6130 100644 --- a/lib/portage/sync/config_checks.py +++ b/lib/portage/sync/config_checks.py @@ -34,7 +34,7 @@ def check_type(repo, logger, module_names): return True -class CheckSyncConfig(object): +class CheckSyncConfig: '''Base repos.conf settings checks class''' def __init__(self, repo=None, logger=None): diff --git a/lib/portage/sync/controller.py b/lib/portage/sync/controller.py index c4c72e73a..6b47ae953 100644 --- a/lib/portage/sync/controller.py +++ b/lib/portage/sync/controller.py @@ -1,4 +1,4 @@ -# Copyright 2014-2019 Gentoo Authors +# Copyright 2014-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -29,7 +29,7 @@ from portage import util from _emerge.CompositeTask import CompositeTask -class TaskHandler(object): +class TaskHandler: """Handles the running of the tasks it is given """ @@ -82,7 +82,7 @@ def print_results(results): print("\n") -class SyncManager(object): +class SyncManager: '''Main sync control module''' def __init__(self, settings, logger): @@ -115,8 +115,7 @@ class SyncManager(object): "has been renamed to sync_async", DeprecationWarning, stacklevel=2) return self.sync_async - else: - raise AttributeError(name) + raise AttributeError(name) def get_module_descriptions(self, mod): desc = self.module_controller.get_func_descriptions(mod) @@ -231,7 +230,7 @@ class SyncManager(object): # Redirect command stderr to stdout, in order to prevent # spurious cron job emails (bug 566132). spawn_kwargs["fd_pipes"] = { - 0: sys.__stdin__.fileno(), + 0: portage._get_stdin().fileno(), 1: sys.__stdout__.fileno(), 2: sys.__stdout__.fileno() } @@ -396,4 +395,3 @@ class SyncRepo(CompositeTask): self.returncode = sync_task.returncode self.sync_callback(self.sync_task) self._async_wait() - diff --git a/lib/portage/sync/getaddrinfo_validate.py b/lib/portage/sync/getaddrinfo_validate.py index 5e6009c74..fd77f4875 100644 --- a/lib/portage/sync/getaddrinfo_validate.py +++ b/lib/portage/sync/getaddrinfo_validate.py @@ -1,10 +1,7 @@ -# Copyright 2010 Gentoo Foundation +# Copyright 2010-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -import sys -if sys.hexversion >= 0x3000000: - basestring = str def getaddrinfo_validate(addrinfos): """ @@ -19,7 +16,7 @@ def getaddrinfo_validate(addrinfos): continue if len(addrinfo[4]) < 2: continue - if not isinstance(addrinfo[4][0], basestring): + if not isinstance(addrinfo[4][0], str): continue except TypeError: continue diff --git a/lib/portage/sync/modules/git/__init__.py b/lib/portage/sync/modules/git/__init__.py index 270d97186..2081d0d25 100644 --- a/lib/portage/sync/modules/git/__init__.py +++ b/lib/portage/sync/modules/git/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2014-2018 Gentoo Foundation +# Copyright 2014-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 doc = """Git plug-in module for portage. @@ -14,6 +14,7 @@ class CheckGitConfig(CheckSyncConfig): def __init__(self, repo, logger): CheckSyncConfig.__init__(self, repo, logger) self.checks.append('check_depth') + self.checks.append('check_verify_commit_signature') def check_depth(self): for attr in ('clone_depth', 'sync_depth'): @@ -33,6 +34,16 @@ class CheckGitConfig(CheckSyncConfig): else: setattr(self.repo, attr, d) + def check_verify_commit_signature(self): + v = self.repo.module_specific_options.get( + 'sync-git-verify-commit-signature', 'false').lower() + + if v not in ('yes', 'no', 'true', 'false'): + writemsg_level("!!! %s\n" % + _("sync-git-verify-commit-signature not one of: %s") + % ('{yes, no, true, false}'), + level=self.logger.ERROR, noiselevel=-1) + module_spec = { 'name': 'git', diff --git a/lib/portage/sync/modules/git/git.py b/lib/portage/sync/modules/git/git.py index 7df4b6d61..d87f1a601 100644 --- a/lib/portage/sync/modules/git/git.py +++ b/lib/portage/sync/modules/git/git.py @@ -1,4 +1,4 @@ -# Copyright 2005-2018 Gentoo Foundation +# Copyright 2005-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import io @@ -204,7 +204,7 @@ class GitSync(NewBase): def verify_head(self, revision='-1'): if (self.repo.module_specific_options.get( - 'sync-git-verify-commit-signature', 'false') != 'true'): + 'sync-git-verify-commit-signature', 'false').lower() not in ('true', 'yes')): return True if self.repo.sync_openpgp_key_path is not None: @@ -247,26 +247,25 @@ class GitSync(NewBase): if status == 'G': # good signature is good out.einfo('Trusted signature found on top commit') return True - elif status == 'U': # untrusted + if status == 'U': # untrusted out.ewarn('Top commit signature is valid but not trusted') return True + if status == 'B': + expl = 'bad signature' + elif status == 'X': + expl = 'expired signature' + elif status == 'Y': + expl = 'expired key' + elif status == 'R': + expl = 'revoked key' + elif status == 'E': + expl = 'unable to verify signature (missing key?)' + elif status == 'N': + expl = 'no signature' else: - if status == 'B': - expl = 'bad signature' - elif status == 'X': - expl = 'expired signature' - elif status == 'Y': - expl = 'expired key' - elif status == 'R': - expl = 'revoked key' - elif status == 'E': - expl = 'unable to verify signature (missing key?)' - elif status == 'N': - expl = 'no signature' - else: - expl = 'unknown issue' - out.eerror('No valid signature found: %s' % (expl,)) - return False + expl = 'unknown issue' + out.eerror('No valid signature found: %s' % (expl,)) + return False finally: if openpgp_env is not None: openpgp_env.close() diff --git a/lib/portage/sync/modules/rsync/rsync.py b/lib/portage/sync/modules/rsync/rsync.py index a40e1c854..33019534b 100644 --- a/lib/portage/sync/modules/rsync/rsync.py +++ b/lib/portage/sync/modules/rsync/rsync.py @@ -1,4 +1,4 @@ -# Copyright 1999-2018 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import sys @@ -10,7 +10,6 @@ import datetime import io import re import random -import subprocess import tempfile import portage @@ -36,11 +35,6 @@ try: except ImportError: gemato = None -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - _unicode = str -else: - _unicode = unicode SERVER_OUT_OF_DATE = -1 EXCEEDED_MAX_RETRIES = -2 @@ -68,7 +62,7 @@ class RsyncSync(NewBase): out = portage.output.EOutput(quiet=quiet) syncuri = self.repo.sync_uri if self.repo.module_specific_options.get( - 'sync-rsync-vcs-ignore', 'false').lower() == 'true': + 'sync-rsync-vcs-ignore', 'false').lower() in ('true', 'yes'): vcs_dirs = () else: vcs_dirs = frozenset(VCS_DIRS) @@ -102,7 +96,7 @@ class RsyncSync(NewBase): # via default repos.conf though. self.verify_metamanifest = ( self.repo.module_specific_options.get( - 'sync-rsync-verify-metamanifest', 'no') in ('yes', 'true')) + 'sync-rsync-verify-metamanifest', 'no').lower() in ('yes', 'true')) # Support overriding job count. self.verify_jobs = self.repo.module_specific_options.get( 'sync-rsync-verify-jobs', None) @@ -243,7 +237,7 @@ class RsyncSync(NewBase): except socket.error as e: writemsg_level( "!!! getaddrinfo failed for '%s': %s\n" - % (_unicode_decode(hostname), _unicode(e)), + % (_unicode_decode(hostname), str(e)), noiselevel=-1, level=logging.ERROR) if addrinfos: diff --git a/lib/portage/sync/modules/webrsync/webrsync.py b/lib/portage/sync/modules/webrsync/webrsync.py index 70f65cfcd..20cc25a2c 100644 --- a/lib/portage/sync/modules/webrsync/webrsync.py +++ b/lib/portage/sync/modules/webrsync/webrsync.py @@ -134,4 +134,3 @@ class PyWebRsync(SyncBase): def sync(self, **kwargs): '''Sync the repository''' pass - diff --git a/lib/portage/sync/syncbase.py b/lib/portage/sync/syncbase.py index 46644d68e..5f18e5ba3 100644 --- a/lib/portage/sync/syncbase.py +++ b/lib/portage/sync/syncbase.py @@ -1,4 +1,4 @@ -# Copyright 2014-2018 Gentoo Foundation +# Copyright 2014-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 ''' @@ -6,7 +6,6 @@ Base class for performing sync operations. This class contains common initialization code and functions. ''' -from __future__ import unicode_literals import functools import logging import os @@ -21,7 +20,7 @@ from portage.util.futures.retry import retry from portage.util.futures.executor.fork import ForkExecutor from . import _SUBMODULE_PATH_MAP -class SyncBase(object): +class SyncBase: '''Base Sync class for subclassing''' short_desc = "Perform sync operations on repositories" @@ -252,6 +251,13 @@ class SyncBase(object): @type openpgp_env: gemato.openpgp.OpenPGPEnvironment """ out = portage.output.EOutput(quiet=('--quiet' in self.options['emerge_config'].opts)) + + if not self.repo.sync_openpgp_key_refresh: + out.ewarn('Key refresh is disabled via a repos.conf sync-openpgp-key-refresh') + out.ewarn('setting, and this is a security vulnerability because it prevents') + out.ewarn('detection of revoked keys!') + return + out.ebegin('Refreshing keys via WKD') if openpgp_env.refresh_keys_wkd(): out.eend(0) diff --git a/lib/portage/tests/bin/setup_env.py b/lib/portage/tests/bin/setup_env.py index 33b167989..83d99528e 100644 --- a/lib/portage/tests/bin/setup_env.py +++ b/lib/portage/tests/bin/setup_env.py @@ -4,6 +4,7 @@ import tempfile +import portage from portage import os from portage import shutil from portage.const import PORTAGE_BIN_PATH @@ -36,6 +37,7 @@ def binTestsInit(): env['PATH'] = bindir + ':' + os.environ['PATH'] env['PORTAGE_BIN_PATH'] = bindir env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH + env['PORTAGE_PYTHON'] = portage._python_interpreter env['PORTAGE_INST_UID'] = str(os.getuid()) env['PORTAGE_INST_GID'] = str(os.getgid()) env['DESTTREE'] = '/usr' @@ -71,17 +73,17 @@ def portage_func(func, args, exit_status=0): fd_pipes=fd_pipes, pre_exec=pre_exec) f.close() -def create_portage_wrapper(bin): +def create_portage_wrapper(f): def derived_func(*args): newargs = list(args) - newargs.insert(0, bin) + newargs.insert(0, f) return portage_func(*newargs) return derived_func -for bin in os.listdir(os.path.join(bindir, 'ebuild-helpers')): - if bin.startswith('do') or \ - bin.startswith('new') or \ - bin.startswith('prep') or \ - bin in ('fowners', 'fperms'): - globals()[bin] = create_portage_wrapper( - os.path.join(bindir, 'ebuild-helpers', bin)) +for f in os.listdir(os.path.join(bindir, 'ebuild-helpers')): + if (f.startswith('do') or + f.startswith('new') or + f.startswith('prep') or + f in ('fowners', 'fperms')): + globals()[f] = create_portage_wrapper( + os.path.join(bindir, 'ebuild-helpers', f)) diff --git a/lib/portage/tests/dbapi/test_auxdb.py b/lib/portage/tests/dbapi/test_auxdb.py index 85d64c15e..5c79357d7 100644 --- a/lib/portage/tests/dbapi/test_auxdb.py +++ b/lib/portage/tests/dbapi/test_auxdb.py @@ -1,8 +1,6 @@ # Copyright 2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - from portage.tests import TestCase from portage.tests.resolver.ResolverPlayground import ResolverPlayground from portage.util.futures import asyncio diff --git a/lib/portage/tests/dep/testAtom.py b/lib/portage/tests/dep/testAtom.py index 4ee06e0a2..b454782d1 100644 --- a/lib/portage/tests/dep/testAtom.py +++ b/lib/portage/tests/dep/testAtom.py @@ -257,7 +257,7 @@ class TestAtom(TestCase): ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", [], ["a", "b", "c", "d", "e", "f"], None), ) - class use_flag_validator(object): + class use_flag_validator: def __init__(self, iuse): self.iuse = iuse diff --git a/lib/portage/tests/dep/test_isvalidatom.py b/lib/portage/tests/dep/test_isvalidatom.py index 9d3367aab..58d999646 100644 --- a/lib/portage/tests/dep/test_isvalidatom.py +++ b/lib/portage/tests/dep/test_isvalidatom.py @@ -4,7 +4,7 @@ from portage.tests import TestCase from portage.dep import isvalidatom -class IsValidAtomTestCase(object): +class IsValidAtomTestCase: def __init__(self, atom, expected, allow_wildcard=False, allow_repo=False, allow_build_id=False): self.atom = atom diff --git a/lib/portage/tests/dep/test_match_from_list.py b/lib/portage/tests/dep/test_match_from_list.py index 3080479c2..35ba3a445 100644 --- a/lib/portage/tests/dep/test_match_from_list.py +++ b/lib/portage/tests/dep/test_match_from_list.py @@ -1,16 +1,12 @@ -# Copyright 2006-2014 Gentoo Foundation +# Copyright 2006-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -import sys from portage.tests import TestCase from portage.dep import Atom, match_from_list, _repo_separator from portage.versions import catpkgsplit, _pkg_str -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str -class Package(object): +class Package: """ Provides a minimal subset of attributes of _emerge.Package.Package """ @@ -34,16 +30,16 @@ class Package(object): self.use = self._use_class([]) self.iuse = self._iuse_class([]) - class _use_class(object): + class _use_class: def __init__(self, use): self.enabled = frozenset(use) - class _iuse_class(object): + class _iuse_class: def __init__(self, iuse): self.all = frozenset(iuse) def is_valid_flag(self, flags): - if isinstance(flags, basestring): + if isinstance(flags, str): flags = [flags] for flag in flags: if not flag in self.all: diff --git a/lib/portage/tests/dep/test_soname_atom_pickle.py b/lib/portage/tests/dep/test_soname_atom_pickle.py index c3f339e2d..03a4893d1 100644 --- a/lib/portage/tests/dep/test_soname_atom_pickle.py +++ b/lib/portage/tests/dep/test_soname_atom_pickle.py @@ -1,9 +1,6 @@ # Copyright 2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - -import sys from portage.dep.soname.SonameAtom import SonameAtom from portage.tests import TestCase diff --git a/lib/portage/tests/dep/test_use_reduce.py b/lib/portage/tests/dep/test_use_reduce.py index d9ee5a309..316f28952 100644 --- a/lib/portage/tests/dep/test_use_reduce.py +++ b/lib/portage/tests/dep/test_use_reduce.py @@ -5,7 +5,7 @@ from portage.tests import TestCase from portage.exception import InvalidDependString from portage.dep import Atom, use_reduce -class UseReduceTestCase(object): +class UseReduceTestCase: def __init__(self, deparray, uselist=[], masklist=[], matchall=0, excludeall=[], is_src_uri=False, eapi='0', opconvert=False, flat=False, expected_result=None, diff --git a/lib/portage/tests/ebuild/test_config.py b/lib/portage/tests/ebuild/test_config.py index dcb5ffe0d..60f3ecab4 100644 --- a/lib/portage/tests/ebuild/test_config.py +++ b/lib/portage/tests/ebuild/test_config.py @@ -1,8 +1,6 @@ # Copyright 2010-2015 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - import io import tempfile diff --git a/lib/portage/tests/ebuild/test_fetch.py b/lib/portage/tests/ebuild/test_fetch.py index 9a8a4a544..5b67dc519 100644 --- a/lib/portage/tests/ebuild/test_fetch.py +++ b/lib/portage/tests/ebuild/test_fetch.py @@ -1,8 +1,6 @@ -# Copyright 2019 Gentoo Authors +# Copyright 2019-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - import functools import io import tempfile @@ -169,7 +167,7 @@ class EbuildFetchTestCase(TestCase): '--repositories-configuration', settings.repositories.config_string(), '--repo', 'test_repo', '--mirror') - env = os.environ.copy() + env = settings.environ() env['PYTHONPATH'] = ':'.join( filter(None, [PORTAGE_PYM_PATH] + os.environ.get('PYTHONPATH', '').split(':'))) diff --git a/lib/portage/tests/ebuild/test_spawn.py b/lib/portage/tests/ebuild/test_spawn.py index a38e10972..2e2b47eb9 100644 --- a/lib/portage/tests/ebuild/test_spawn.py +++ b/lib/portage/tests/ebuild/test_spawn.py @@ -3,7 +3,6 @@ import errno import io -import sys import tempfile import portage from portage import os diff --git a/lib/portage/tests/ebuild/test_use_expand_incremental.py b/lib/portage/tests/ebuild/test_use_expand_incremental.py index a58f08cb9..4e83e4f85 100644 --- a/lib/portage/tests/ebuild/test_use_expand_incremental.py +++ b/lib/portage/tests/ebuild/test_use_expand_incremental.py @@ -1,8 +1,6 @@ # Copyright 2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - import io from portage import os, _encodings diff --git a/lib/portage/tests/emerge/test_config_protect.py b/lib/portage/tests/emerge/test_config_protect.py index 06ab059ee..f88cb3b7b 100644 --- a/lib/portage/tests/emerge/test_config_protect.py +++ b/lib/portage/tests/emerge/test_config_protect.py @@ -1,8 +1,6 @@ # Copyright 2014-2015 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - import io from functools import partial import shutil diff --git a/lib/portage/tests/env/__init__.py b/lib/portage/tests/env/__init__.py index cbeabe5c6..59a3a1e22 100644 --- a/lib/portage/tests/env/__init__.py +++ b/lib/portage/tests/env/__init__.py @@ -1,4 +1,3 @@ # tests/portage/env/__init__.py -- Portage Unit Test functionality # Copyright 2007 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 - diff --git a/lib/portage/tests/env/config/__init__.py b/lib/portage/tests/env/config/__init__.py index ef5cc43b6..daa503ba3 100644 --- a/lib/portage/tests/env/config/__init__.py +++ b/lib/portage/tests/env/config/__init__.py @@ -1,4 +1,3 @@ # tests/portage/env/config/__init__.py -- Portage Unit Test functionality # Copyright 2007 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 - diff --git a/lib/portage/tests/glsa/test_security_set.py b/lib/portage/tests/glsa/test_security_set.py index a602f83d4..025fcd864 100644 --- a/lib/portage/tests/glsa/test_security_set.py +++ b/lib/portage/tests/glsa/test_security_set.py @@ -1,8 +1,6 @@ # Copyright 2013-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - import io import portage diff --git a/lib/portage/tests/lint/test_bash_syntax.py b/lib/portage/tests/lint/test_bash_syntax.py index fdbb6fe88..da6ee4f45 100644 --- a/lib/portage/tests/lint/test_bash_syntax.py +++ b/lib/portage/tests/lint/test_bash_syntax.py @@ -4,7 +4,6 @@ from itertools import chain import stat import subprocess -import sys from portage.const import BASH_BINARY, PORTAGE_BASE_PATH, PORTAGE_BIN_PATH from portage.tests import TestCase diff --git a/lib/portage/tests/process/test_AsyncFunction.py b/lib/portage/tests/process/test_AsyncFunction.py new file mode 100644 index 000000000..55857026d --- /dev/null +++ b/lib/portage/tests/process/test_AsyncFunction.py @@ -0,0 +1,38 @@ +# Copyright 2020 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +import sys + +from portage import os +from portage.tests import TestCase +from portage.util._async.AsyncFunction import AsyncFunction +from portage.util.futures import asyncio +from portage.util.futures._asyncio.streams import _writer +from portage.util.futures.compat_coroutine import coroutine +from portage.util.futures.unix_events import _set_nonblocking + + +class AsyncFunctionTestCase(TestCase): + + @staticmethod + def _read_from_stdin(pw): + os.close(pw) + return ''.join(sys.stdin) + + @coroutine + def _testAsyncFunctionStdin(self, loop): + test_string = '1\n2\n3\n' + pr, pw = os.pipe() + fd_pipes = {0:pr} + reader = AsyncFunction(scheduler=loop, fd_pipes=fd_pipes, target=self._read_from_stdin, args=(pw,)) + reader.start() + os.close(pr) + _set_nonblocking(pw) + with open(pw, mode='wb', buffering=0) as pipe_write: + yield _writer(pipe_write, test_string.encode('utf_8'), loop=loop) + self.assertEqual((yield reader.async_wait()), os.EX_OK) + self.assertEqual(reader.result, test_string) + + def testAsyncFunctionStdin(self): + loop = asyncio._wrap_loop() + loop.run_until_complete(self._testAsyncFunctionStdin(loop)) diff --git a/lib/portage/tests/process/test_PipeLogger.py b/lib/portage/tests/process/test_PipeLogger.py new file mode 100644 index 000000000..2bd94cf39 --- /dev/null +++ b/lib/portage/tests/process/test_PipeLogger.py @@ -0,0 +1,58 @@ +# Copyright 2020 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +from portage import os +from portage.tests import TestCase +from portage.util._async.PipeLogger import PipeLogger +from portage.util.futures import asyncio +from portage.util.futures._asyncio.streams import _reader, _writer +from portage.util.futures.compat_coroutine import coroutine, coroutine_return +from portage.util.futures.unix_events import _set_nonblocking + + +class PipeLoggerTestCase(TestCase): + + @coroutine + def _testPipeLoggerToPipe(self, test_string, loop=None): + """ + Test PipeLogger writing to a pipe connected to a PipeReader. + This verifies that PipeLogger does not deadlock when writing + to a pipe that's drained by a PipeReader running in the same + process (requires non-blocking write). + """ + + input_fd, writer_pipe = os.pipe() + _set_nonblocking(writer_pipe) + writer_pipe = os.fdopen(writer_pipe, 'wb', 0) + writer = asyncio.ensure_future(_writer(writer_pipe, test_string.encode('ascii'), loop=loop), loop=loop) + writer.add_done_callback(lambda writer: writer_pipe.close()) + + pr, pw = os.pipe() + + consumer = PipeLogger(background=True, + input_fd=input_fd, + log_file_path=os.fdopen(pw, 'wb', 0), + scheduler=loop) + consumer.start() + + # Before starting the reader, wait here for a moment, in order + # to exercise PipeLogger's handling of EAGAIN during write. + yield asyncio.wait([writer], timeout=0.01) + + reader = _reader(pr, loop=loop) + yield writer + content = yield reader + yield consumer.async_wait() + + self.assertEqual(consumer.returncode, os.EX_OK) + + coroutine_return(content.decode('ascii', 'replace')) + + def testPipeLogger(self): + loop = asyncio._wrap_loop() + + for x in (1, 2, 5, 6, 7, 8, 2**5, 2**10, 2**12, 2**13, 2**14, 2**17, 2**17 + 1): + test_string = x * "a" + output = loop.run_until_complete(self._testPipeLoggerToPipe(test_string, loop=loop)) + self.assertEqual(test_string, output, + "x = %s, len(output) = %s" % (x, len(output))) diff --git a/lib/portage/tests/process/test_poll.py b/lib/portage/tests/process/test_poll.py index f505b5049..dc11f6d13 100644 --- a/lib/portage/tests/process/test_poll.py +++ b/lib/portage/tests/process/test_poll.py @@ -1,11 +1,10 @@ -# Copyright 1998-2019 Gentoo Authors +# Copyright 1998-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import functools import pty import shutil import socket -import sys import tempfile from portage import os @@ -34,11 +33,8 @@ class PipeReaderTestCase(TestCase): def test_domain_socket(self): def make_pipes(): - if sys.version_info >= (3, 2): - read_end, write_end = socket.socketpair() - return (read_end.detach(), write_end.detach()), None - else: - self.skipTest('socket detach not supported') + read_end, write_end = socket.socketpair() + return (read_end.detach(), write_end.detach()), None self._do_test(make_pipes) def test_named_pipe(self): diff --git a/lib/portage/tests/resolver/ResolverPlayground.py b/lib/portage/tests/resolver/ResolverPlayground.py index 0751e392e..8996eba92 100644 --- a/lib/portage/tests/resolver/ResolverPlayground.py +++ b/lib/portage/tests/resolver/ResolverPlayground.py @@ -4,7 +4,6 @@ import bz2 from itertools import permutations import fnmatch -import sys import tempfile import portage from portage import os @@ -34,11 +33,8 @@ try: except ImportError: cnf_path_repoman = None -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str -class ResolverPlayground(object): +class ResolverPlayground: """ This class helps to create the necessary files on disk and the needed settings instances, etc. for the resolver to do @@ -91,6 +87,7 @@ class ResolverPlayground(object): "chgrp", "chmod", "chown", + "comm", "cp", "egrep", "env", @@ -411,7 +408,7 @@ class ResolverPlayground(object): for eclass_name, eclass_content in eclasses.items(): with open(os.path.join(eclass_dir, "{}.eclass".format(eclass_name)), 'wt') as f: - if isinstance(eclass_content, basestring): + if isinstance(eclass_content, str): eclass_content = [eclass_content] for line in eclass_content: f.write("{}\n".format(line)) @@ -644,7 +641,7 @@ class ResolverPlayground(object): else: shutil.rmtree(self.eroot) -class ResolverPlaygroundTestCase(object): +class ResolverPlaygroundTestCase: def __init__(self, request, **kwargs): self.all_permutations = kwargs.pop("all_permutations", False) @@ -695,7 +692,7 @@ class ResolverPlaygroundTestCase(object): if expected: new_expected = [] for obj in expected: - if isinstance(obj, basestring): + if isinstance(obj, str): if obj[:1] == "!": new_expected.append(obj) continue @@ -720,7 +717,7 @@ class ResolverPlaygroundTestCase(object): while got_stack and expected_stack: got_token = got_stack.pop() expected_obj = expected_stack.pop() - if isinstance(expected_obj, basestring): + if isinstance(expected_obj, str): new_expected.append(expected_obj) if got_token == expected_obj: continue @@ -822,7 +819,7 @@ def _mergelist_str(x, depgraph): return mergelist_str -class ResolverPlaygroundResult(object): +class ResolverPlaygroundResult: checks = ( "success", "mergelist", "use_changes", "license_changes", @@ -916,7 +913,7 @@ class ResolverPlaygroundResult(object): if required_use_unsatisfied: self.required_use_unsatisfied = set(required_use_unsatisfied) -class ResolverPlaygroundDepcleanResult(object): +class ResolverPlaygroundDepcleanResult: checks = ( "success", "cleanlist", "ordered", "req_pkg_count", diff --git a/lib/portage/tests/resolver/test_binary_pkg_ebuild_visibility.py b/lib/portage/tests/resolver/test_binary_pkg_ebuild_visibility.py index 0d01d0696..0cd650402 100644 --- a/lib/portage/tests/resolver/test_binary_pkg_ebuild_visibility.py +++ b/lib/portage/tests/resolver/test_binary_pkg_ebuild_visibility.py @@ -141,4 +141,3 @@ class BinaryPkgEbuildVisibilityTestCase(TestCase): test_case.fail_msg) finally: playground.cleanup() - diff --git a/lib/portage/tests/resolver/test_profile_default_eapi.py b/lib/portage/tests/resolver/test_profile_default_eapi.py index cc5721949..130c231fd 100644 --- a/lib/portage/tests/resolver/test_profile_default_eapi.py +++ b/lib/portage/tests/resolver/test_profile_default_eapi.py @@ -1,8 +1,6 @@ # Copyright 2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - import io from portage import os, _encodings diff --git a/lib/portage/tests/resolver/test_profile_package_set.py b/lib/portage/tests/resolver/test_profile_package_set.py index 88a2a8259..a11609832 100644 --- a/lib/portage/tests/resolver/test_profile_package_set.py +++ b/lib/portage/tests/resolver/test_profile_package_set.py @@ -1,8 +1,6 @@ # Copyright 2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - import io from portage import os, _encodings diff --git a/lib/portage/tests/sets/files/testConfigFileSet.py b/lib/portage/tests/sets/files/testConfigFileSet.py index 3ec26a077..395da062a 100644 --- a/lib/portage/tests/sets/files/testConfigFileSet.py +++ b/lib/portage/tests/sets/files/testConfigFileSet.py @@ -29,4 +29,3 @@ class ConfigFileSetTestCase(TestCase): s = ConfigFileSet(self.testfile) s.load() self.assertEqual(set(test_cps), s.getAtoms()) - diff --git a/lib/portage/tests/sets/files/testStaticFileSet.py b/lib/portage/tests/sets/files/testStaticFileSet.py index d515a6728..11fc9a9d8 100644 --- a/lib/portage/tests/sets/files/testStaticFileSet.py +++ b/lib/portage/tests/sets/files/testStaticFileSet.py @@ -24,4 +24,3 @@ class StaticFileSetTestCase(TestCase): s = StaticFileSet(self.testfile) s.load() self.assertEqual(set(test_cps), s.getAtoms()) - diff --git a/lib/portage/tests/sets/shell/testShell.py b/lib/portage/tests/sets/shell/testShell.py index 2cdd833c3..58dc750bf 100644 --- a/lib/portage/tests/sets/shell/testShell.py +++ b/lib/portage/tests/sets/shell/testShell.py @@ -17,12 +17,12 @@ class CommandOutputSetTestCase(TestCase): def testCommand(self): - input = set(test_cps) + params = set(test_cps) command = find_binary("bash") command += " -c '" - for a in input: - command += " echo -e \"%s\" ; " % a + for a in params: + command += " echo -e \"%s\" ; " % a command += "'" s = CommandOutputSet(command) atoms = s.getAtoms() - self.assertEqual(atoms, input) + self.assertEqual(atoms, params) diff --git a/lib/portage/tests/sync/test_sync_local.py b/lib/portage/tests/sync/test_sync_local.py index 5fb8afb7c..efd61aac8 100644 --- a/lib/portage/tests/sync/test_sync_local.py +++ b/lib/portage/tests/sync/test_sync_local.py @@ -1,11 +1,10 @@ -# Copyright 2014-2015 Gentoo Foundation +# Copyright 2014-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import datetime import subprocess import sys import textwrap -import time import portage from portage import os, shutil, _shell_quote @@ -72,6 +71,7 @@ class SyncLocalTestCase(TestCase): distdir = os.path.join(eprefix, "distdir") repo = settings.repositories["test_repo"] metadata_dir = os.path.join(repo.location, "metadata") + rcu_store_dir = os.path.join(eprefix, 'var/repositories/test_repo_rcu_storedir') cmds = {} for cmd in ("emerge", "emaint"): @@ -192,6 +192,10 @@ class SyncLocalTestCase(TestCase): (homedir, lambda: os.mkdir(repo.user_location)), ) + delete_rcu_store_dir = ( + (homedir, lambda: shutil.rmtree(rcu_store_dir)), + ) + revert_rcu_layout = ( (homedir, lambda: os.rename(repo.user_location, repo.user_location + '.bak')), (homedir, lambda: os.rename(os.path.realpath(repo.user_location + '.bak'), repo.user_location)), @@ -290,7 +294,8 @@ class SyncLocalTestCase(TestCase): for cwd, cmd in rename_repo + sync_cmds_auto_sync + sync_cmds + \ rsync_opts_repos + rsync_opts_repos_default + \ rsync_opts_repos_default_ovr + rsync_opts_repos_default_cancel + \ - bump_timestamp_cmds + sync_rsync_rcu + sync_cmds + revert_rcu_layout + \ + bump_timestamp_cmds + sync_rsync_rcu + sync_cmds + delete_rcu_store_dir + \ + sync_cmds + revert_rcu_layout + \ delete_repo_location + sync_cmds + sync_cmds + \ bump_timestamp_cmds + sync_cmds + revert_rcu_layout + \ delete_sync_repo + git_repo_create + sync_type_git + \ diff --git a/lib/portage/tests/unicode/test_string_format.py b/lib/portage/tests/unicode/test_string_format.py index 9d4366a91..3b994d622 100644 --- a/lib/portage/tests/unicode/test_string_format.py +++ b/lib/portage/tests/unicode/test_string_format.py @@ -1,9 +1,6 @@ -# Copyright 2010-2014 Gentoo Foundation +# Copyright 2010-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - -import sys from portage import _encodings, _unicode_encode from portage.exception import PortageException @@ -11,11 +8,6 @@ from portage.tests import TestCase from _emerge.DependencyArg import DependencyArg from _emerge.UseFlagDisplay import UseFlagDisplay -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str - -STR_IS_UNICODE = sys.hexversion >= 0x3000000 class StringFormatTestCase(TestCase): """ @@ -44,17 +36,9 @@ class StringFormatTestCase(TestCase): formatted_str = "%s" % (dependency_arg,) self.assertEqual(formatted_str, arg_unicode) - if STR_IS_UNICODE: - - # Test the __str__ method which returns unicode in python3 - formatted_str = "%s" % (dependency_arg,) - self.assertEqual(formatted_str, arg_unicode) - - else: - - # Test the __str__ method which returns encoded bytes in python2 - formatted_bytes = b"%s" % (dependency_arg,) - self.assertEqual(formatted_bytes, arg_bytes) + # Test the __str__ method which returns unicode in python3 + formatted_str = "%s" % (dependency_arg,) + self.assertEqual(formatted_str, arg_unicode) def testPortageException(self): @@ -69,17 +53,9 @@ class StringFormatTestCase(TestCase): formatted_str = "%s" % (e,) self.assertEqual(formatted_str, arg_unicode) - if STR_IS_UNICODE: - - # Test the __str__ method which returns unicode in python3 - formatted_str = "%s" % (e,) - self.assertEqual(formatted_str, arg_unicode) - - else: - - # Test the __str__ method which returns encoded bytes in python2 - formatted_bytes = b"%s" % (e,) - self.assertEqual(formatted_bytes, arg_bytes) + # Test the __str__ method which returns unicode in python3 + formatted_str = "%s" % (e,) + self.assertEqual(formatted_str, arg_unicode) def testUseFlagDisplay(self): @@ -93,16 +69,8 @@ class StringFormatTestCase(TestCase): # Use unicode_literals for unicode format string so that # __unicode__() is called in Python 2. formatted_str = "%s" % (e,) - self.assertEqual(isinstance(formatted_str, basestring), True) + self.assertEqual(isinstance(formatted_str, str), True) - if STR_IS_UNICODE: - - # Test the __str__ method which returns unicode in python3 - formatted_str = "%s" % (e,) - self.assertEqual(isinstance(formatted_str, str), True) - - else: - - # Test the __str__ method which returns encoded bytes in python2 - formatted_bytes = b"%s" % (e,) - self.assertEqual(isinstance(formatted_bytes, bytes), True) + # Test the __str__ method which returns unicode in python3 + formatted_str = "%s" % (e,) + self.assertEqual(isinstance(formatted_str, str), True) diff --git a/lib/portage/tests/util/__init__.py b/lib/portage/tests/util/__init__.py index 69ce1898d..858d731e4 100644 --- a/lib/portage/tests/util/__init__.py +++ b/lib/portage/tests/util/__init__.py @@ -1,4 +1,3 @@ # tests/portage.util/__init__.py -- Portage Unit Test functionality # Copyright 2006 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 - diff --git a/lib/portage/tests/util/futures/asyncio/test_pipe_closed.py b/lib/portage/tests/util/futures/asyncio/test_pipe_closed.py index 507385c04..d9ada8b70 100644 --- a/lib/portage/tests/util/futures/asyncio/test_pipe_closed.py +++ b/lib/portage/tests/util/futures/asyncio/test_pipe_closed.py @@ -1,4 +1,4 @@ -# Copyright 2018 Gentoo Foundation +# Copyright 2018-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import errno @@ -6,7 +6,6 @@ import os import pty import shutil import socket -import sys import tempfile from portage.tests import TestCase @@ -18,7 +17,7 @@ from portage.util.futures.unix_events import ( ) -class _PipeClosedTestCase(object): +class _PipeClosedTestCase: def test_pipe(self): read_end, write_end = os.pipe() @@ -32,10 +31,7 @@ class _PipeClosedTestCase(object): self._do_test(read_end, write_end) def test_domain_socket(self): - if sys.version_info >= (3, 2): - read_end, write_end = socket.socketpair() - else: - self.skipTest('socket detach not supported') + read_end, write_end = socket.socketpair() self._do_test(read_end.detach(), write_end.detach()) def test_named_pipe(self): diff --git a/lib/portage/tests/util/futures/asyncio/test_policy_wrapper_recursion.py b/lib/portage/tests/util/futures/asyncio/test_policy_wrapper_recursion.py index d3cd94b35..900bb5115 100644 --- a/lib/portage/tests/util/futures/asyncio/test_policy_wrapper_recursion.py +++ b/lib/portage/tests/util/futures/asyncio/test_policy_wrapper_recursion.py @@ -1,10 +1,7 @@ # Copyright 2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -try: - import asyncio -except ImportError: - asyncio = None +import asyncio from portage.tests import TestCase from portage.util.futures.unix_events import DefaultEventLoopPolicy @@ -12,9 +9,6 @@ from portage.util.futures.unix_events import DefaultEventLoopPolicy class PolicyWrapperRecursionTestCase(TestCase): def testPolicyWrapperRecursion(self): - if asyncio is None: - self.skipTest('asyncio is not available') - initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) diff --git a/lib/portage/tests/util/futures/asyncio/test_subprocess_exec.py b/lib/portage/tests/util/futures/asyncio/test_subprocess_exec.py index d7e94d132..6128a7d06 100644 --- a/lib/portage/tests/util/futures/asyncio/test_subprocess_exec.py +++ b/lib/portage/tests/util/futures/asyncio/test_subprocess_exec.py @@ -3,7 +3,6 @@ import os import subprocess -import sys from portage.process import find_binary from portage.tests import TestCase @@ -132,8 +131,6 @@ class SubprocessExecTestCase(TestCase): requires an AbstractEventLoop.connect_read_pipe implementation (and a ReadTransport implementation for it to return). """ - if sys.version_info.major < 3: - self.skipTest('ReadTransport not implemented for python2') args_tuple = (b'hello', b'world') echo_binary = find_binary("echo") @@ -162,8 +159,6 @@ class SubprocessExecTestCase(TestCase): requires an AbstractEventLoop.connect_write_pipe implementation (and a WriteTransport implementation for it to return). """ - if sys.version_info.major < 3: - self.skipTest('WriteTransport not implemented for python2') stdin_data = b'hello world' cat_binary = find_binary("cat") diff --git a/lib/portage/tests/util/futures/test_compat_coroutine.py b/lib/portage/tests/util/futures/test_compat_coroutine.py index b561c0227..5a8230432 100644 --- a/lib/portage/tests/util/futures/test_compat_coroutine.py +++ b/lib/portage/tests/util/futures/test_compat_coroutine.py @@ -123,7 +123,7 @@ class CompatCoroutineTestCase(TestCase): def test_method_coroutine(self): - class Cubby(object): + class Cubby: _empty = object() diff --git a/lib/portage/tests/util/futures/test_retry.py b/lib/portage/tests/util/futures/test_retry.py index 7a1e76280..ce5fb3e11 100644 --- a/lib/portage/tests/util/futures/test_retry.py +++ b/lib/portage/tests/util/futures/test_retry.py @@ -1,17 +1,14 @@ -# Copyright 2018 Gentoo Foundation +# Copyright 2018-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -try: - from concurrent.futures import ThreadPoolExecutor -except ImportError: - ThreadPoolExecutor = None +from concurrent.futures import ThreadPoolExecutor try: import threading except ImportError: import dummy_threading as threading -import sys +import time from portage.tests import TestCase from portage.util._eventloop.global_event_loop import global_event_loop @@ -19,24 +16,23 @@ from portage.util.backoff import RandomExponentialBackoff from portage.util.futures import asyncio from portage.util.futures.retry import retry from portage.util.futures.executor.fork import ForkExecutor -from portage.util.monotonic import monotonic class SucceedLaterException(Exception): pass -class SucceedLater(object): +class SucceedLater: """ A callable object that succeeds some duration of time has passed. """ def __init__(self, duration): - self._succeed_time = monotonic() + duration + self._succeed_time = time.monotonic() + duration def __call__(self): loop = global_event_loop() result = loop.create_future() - remaining = self._succeed_time - monotonic() + remaining = self._succeed_time - time.monotonic() if remaining > 0: loop.call_soon_threadsafe(lambda: None if result.done() else result.set_exception(SucceedLaterException( @@ -51,7 +47,7 @@ class SucceedNeverException(Exception): pass -class SucceedNever(object): +class SucceedNever: """ A callable object that never succeeds. """ @@ -63,7 +59,7 @@ class SucceedNever(object): return result -class HangForever(object): +class HangForever: """ A callable object that sleeps forever. """ @@ -207,12 +203,12 @@ class RetryForkExecutorTestCase(RetryTestCase): lambda kill_switch: event.set()) event.wait() return result.result() - else: - # child process - try: - return loop.run_until_complete(coroutine_func()) - finally: - loop.close() + + # child process + try: + return loop.run_until_complete(coroutine_func()) + finally: + loop.close() def execute_wrapper(): kill_switch = parent_loop.create_future() @@ -229,6 +225,4 @@ class RetryForkExecutorTestCase(RetryTestCase): class RetryThreadExecutorTestCase(RetryForkExecutorTestCase): def _setUpExecutor(self): - if sys.version_info.major < 3: - self.skipTest('ThreadPoolExecutor not supported for python2') self._executor = ThreadPoolExecutor(max_workers=1) diff --git a/lib/portage/tests/util/test_socks5.py b/lib/portage/tests/util/test_socks5.py index 5db85b0a6..ca32651a7 100644 --- a/lib/portage/tests/util/test_socks5.py +++ b/lib/portage/tests/util/test_socks5.py @@ -2,11 +2,9 @@ # Distributed under the terms of the GNU General Public License v2 import functools -import platform import shutil import socket import struct -import sys import tempfile import time @@ -16,15 +14,8 @@ from portage.util._eventloop.global_event_loop import global_event_loop from portage.util import socks5 from portage.const import PORTAGE_BIN_PATH -try: - from http.server import BaseHTTPRequestHandler, HTTPServer -except ImportError: - from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer - -try: - from urllib.request import urlopen -except ImportError: - from urllib import urlopen +from http.server import BaseHTTPRequestHandler, HTTPServer +from urllib.request import urlopen class _Handler(BaseHTTPRequestHandler): @@ -58,7 +49,7 @@ class _Handler(BaseHTTPRequestHandler): pass -class AsyncHTTPServer(object): +class AsyncHTTPServer: def __init__(self, host, content, loop): self._host = host self._content = content @@ -193,19 +184,13 @@ class Socks5ServerTestCase(TestCase): 'PORTAGE_BIN_PATH': PORTAGE_BIN_PATH, } - try: - proxy = socks5.get_socks5_proxy(settings) - except NotImplementedError: - # bug 658172 for python2.7 - self.skipTest('get_socks5_proxy not implemented for {} {}.{}'.format( - platform.python_implementation(), *sys.version_info[:2])) - else: - loop.run_until_complete(socks5.proxy.ready()) + proxy = socks5.get_socks5_proxy(settings) + loop.run_until_complete(socks5.proxy.ready()) - result = loop.run_until_complete(loop.run_in_executor(None, - self._fetch_via_proxy, proxy, host, server.server_port, path)) + result = loop.run_until_complete(loop.run_in_executor(None, + self._fetch_via_proxy, proxy, host, server.server_port, path)) - self.assertEqual(result, content) + self.assertEqual(result, content) finally: socks5.proxy.stop() shutil.rmtree(tempdir) diff --git a/lib/portage/tests/util/test_xattr.py b/lib/portage/tests/util/test_xattr.py index 2e2564a6e..8e8c2a3d6 100644 --- a/lib/portage/tests/util/test_xattr.py +++ b/lib/portage/tests/util/test_xattr.py @@ -5,16 +5,7 @@ from __future__ import print_function -try: - # Try python-3.3 module first. - # pylint: disable=no-name-in-module - from unittest import mock -except ImportError: - try: - # Try standalone module. - import mock - except ImportError: - mock = None +from unittest import mock import subprocess @@ -50,9 +41,6 @@ class SystemCommandsTest(TestCase): )) def _setUp(self): - if mock is None: - self.skipTest('need mock for testing') - return _XattrSystemCommands def _testGetBasic(self): diff --git a/lib/portage/update.py b/lib/portage/update.py index 1920d213a..07e977452 100644 --- a/lib/portage/update.py +++ b/lib/portage/update.py @@ -1,8 +1,6 @@ -# Copyright 1999-2014 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - import errno import io import re @@ -27,12 +25,6 @@ from portage.eapi import _get_eapi_attrs from portage.exception import DirectoryNotFound, InvalidAtom, PortageException from portage.localization import _ -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - long = int - _unicode = str -else: - _unicode = unicode ignored_dbentries = ("CONTENTS", "environment.bz2") @@ -42,8 +34,8 @@ def update_dbentry(update_cmd, mycontent, eapi=None, parent=None): eapi = parent.eapi if update_cmd[0] == "move": - old_value = _unicode(update_cmd[1]) - new_value = _unicode(update_cmd[2]) + old_value = str(update_cmd[1]) + new_value = str(update_cmd[2]) # Use isvalidatom() to check if this move is valid for the # EAPI (characters allowed in package names may vary). @@ -70,7 +62,7 @@ def update_dbentry(update_cmd, mycontent, eapi=None, parent=None): match_from_list(new_atom, [parent]): continue - split_content[i] = _unicode(new_atom) + split_content[i] = str(new_atom) modified = True if modified: @@ -197,7 +189,7 @@ def grab_updates(updpath, prev_mtimes=None): mystat = os.stat(file_path) if update_data or \ file_path not in prev_mtimes or \ - long(prev_mtimes[file_path]) != mystat[stat.ST_MTIME]: + int(prev_mtimes[file_path]) != mystat[stat.ST_MTIME]: f = io.open(_unicode_encode(file_path, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace') diff --git a/lib/portage/util/SlotObject.py b/lib/portage/util/SlotObject.py index ba6215874..1f38a6cb8 100644 --- a/lib/portage/util/SlotObject.py +++ b/lib/portage/util/SlotObject.py @@ -1,7 +1,7 @@ # Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -class SlotObject(object): +class SlotObject: __slots__ = ("__weakref__",) def __init__(self, **kwargs): diff --git a/lib/portage/util/_ShelveUnicodeWrapper.py b/lib/portage/util/_ShelveUnicodeWrapper.py deleted file mode 100644 index adbd5199f..000000000 --- a/lib/portage/util/_ShelveUnicodeWrapper.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2013 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -class ShelveUnicodeWrapper(object): - """ - Convert unicode to str and back again, since python-2.x shelve - module doesn't support unicode. - """ - def __init__(self, shelve_instance): - self._shelve = shelve_instance - - def _encode(self, s): - if isinstance(s, unicode): - s = s.encode('utf_8') - return s - - def __len__(self): - return len(self._shelve) - - def __contains__(self, k): - return self._encode(k) in self._shelve - - def __iter__(self): - return self._shelve.__iter__() - - def items(self): - return self._shelve.iteritems() - - def __setitem__(self, k, v): - self._shelve[self._encode(k)] = self._encode(v) - - def __getitem__(self, k): - return self._shelve[self._encode(k)] - - def __delitem__(self, k): - del self._shelve[self._encode(k)] - - def get(self, k, *args): - return self._shelve.get(self._encode(k), *args) - - def close(self): - self._shelve.close() - - def clear(self): - self._shelve.clear() diff --git a/lib/portage/util/__init__.py b/lib/portage/util/__init__.py index 6bff97fb7..322b217be 100644 --- a/lib/portage/util/__init__.py +++ b/lib/portage/util/__init__.py @@ -1,8 +1,6 @@ -# Copyright 2004-2017 Gentoo Foundation +# Copyright 2004-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - __all__ = ['apply_permissions', 'apply_recursive_permissions', 'apply_secpass_permissions', 'apply_stat_permissions', 'atomic_ofstream', 'cmp_sort_key', 'ConfigProtect', 'dump_traceback', 'ensure_dirs', @@ -16,10 +14,7 @@ __all__ = ['apply_permissions', 'apply_recursive_permissions', from copy import deepcopy import errno import io -try: - from itertools import chain, filterfalse -except ImportError: - from itertools import chain, ifilterfalse as filterfalse +from itertools import chain, filterfalse import logging import re import shlex @@ -50,10 +45,6 @@ from portage.proxy.objectproxy import ObjectProxy from portage.cache.mappings import UserDict from portage.const import EPREFIX -if sys.hexversion >= 0x3000000: - _unicode = str -else: - _unicode = unicode noiselimit = 0 @@ -79,7 +70,7 @@ def writemsg(mystr, noiselevel=0, fd=None): else: mystr = _unicode_encode(mystr, encoding=_encodings['stdio'], errors='backslashreplace') - if sys.hexversion >= 0x3000000 and fd in (sys.stdout, sys.stderr): + if fd in (sys.stdout, sys.stderr): fd = fd.buffer fd.write(mystr) fd.flush() @@ -114,7 +105,7 @@ def normalize_path(mypath): We dislike this behavior so we create our own normpath func to fix it. """ - if sys.hexversion >= 0x3000000 and isinstance(mypath, bytes): + if isinstance(mypath, bytes): path_sep = os.path.sep.encode() else: path_sep = os.path.sep @@ -122,8 +113,7 @@ def normalize_path(mypath): if mypath.startswith(path_sep): # posixpath.normpath collapses 3 or more leading slashes to just 1. return os.path.normpath(2*path_sep + mypath) - else: - return os.path.normpath(mypath) + return os.path.normpath(mypath) def grabfile(myfilename, compat_level=0, recursive=0, remember_source_file=False): """This function grabs the lines in a file, normalizes whitespace and returns lines in a list; if a line @@ -259,9 +249,9 @@ def append_repo(atom_list, repo_name, remember_source_file=False): if remember_source_file: return [(atom.repo is not None and atom or atom.with_repo(repo_name), source) \ for atom, source in atom_list] - else: - return [atom.repo is not None and atom or atom.with_repo(repo_name) \ - for atom in atom_list] + + return [atom.repo is not None and atom or atom.with_repo(repo_name) \ + for atom in atom_list] def stack_lists(lists, incremental=1, remember_source_file=False, warn_for_unmatched_removal=False, strict_warn_for_unmatched_removal=False, ignore_repo=False): @@ -338,8 +328,7 @@ def stack_lists(lists, incremental=1, remember_source_file=False, if remember_source_file: return list(new_list.items()) - else: - return list(new_list) + return list(new_list) def grabdict(myfilename, juststrings=0, empty=0, recursive=0, incremental=1, newlines=0): """ @@ -506,7 +495,7 @@ def grabfile_package(myfilename, compatlevel=0, recursive=0, writemsg(_("--- Invalid atom in %s: %s\n") % (source_file, e), noiselevel=-1) else: - if pkg_orig == _unicode(pkg): + if pkg_orig == str(pkg): # normal atom, so return as Atom instance if remember_source_file: atoms.append((pkg, source_file)) @@ -598,19 +587,15 @@ def writedict(mydict, myfilename, writekey=True): lines.append("%s %s\n" % (k, " ".join(v))) write_atomic(myfilename, "".join(lines)) + def shlex_split(s): """ This is equivalent to shlex.split, but if the current interpreter is python2, it temporarily encodes unicode strings to bytes since python2's shlex.split() doesn't handle unicode strings. """ - convert_to_bytes = sys.hexversion < 0x3000000 and not isinstance(s, bytes) - if convert_to_bytes: - s = _unicode_encode(s) - rval = shlex.split(s) - if convert_to_bytes: - rval = [_unicode_decode(x) for x in rval] - return rval + return shlex.split(s) + class _getconfig_shlex(shlex.shlex): @@ -675,15 +660,9 @@ def getconfig(mycfg, tolerant=False, allow_sourcing=False, expand=True, f = None try: - # NOTE: shlex doesn't support unicode objects with Python 2 - # (produces spurious \0 characters). - if sys.hexversion < 0x3000000: - f = open(_unicode_encode(mycfg, - encoding=_encodings['fs'], errors='strict'), 'rb') - else: - f = open(_unicode_encode(mycfg, - encoding=_encodings['fs'], errors='strict'), mode='r', - encoding=_encodings['content'], errors='replace') + f = open(_unicode_encode(mycfg, + encoding=_encodings['fs'], errors='strict'), mode='r', + encoding=_encodings['content'], errors='replace') content = f.read() except IOError as e: if e.errno == PermissionDenied.errno: @@ -888,9 +867,8 @@ def varexpand(mystring, mydict=None, error_leader=None): msg = error_leader() + msg writemsg(msg + "\n", noiselevel=-1) return "" - else: - pos += 1 - break + pos += 1 + break pos += 1 myvarname = mystring[myvstart:pos] if braced: @@ -900,8 +878,7 @@ def varexpand(mystring, mydict=None, error_leader=None): msg = error_leader() + msg writemsg(msg + "\n", noiselevel=-1) return "" - else: - pos += 1 + pos += 1 if len(myvarname) == 0: msg = "$" if braced: @@ -962,7 +939,7 @@ def dump_traceback(msg, noiselevel=1): writemsg(error+"\n", noiselevel=noiselevel) writemsg("====================================\n\n", noiselevel=noiselevel) -class cmp_sort_key(object): +class cmp_sort_key: """ In python-3.0 the list.sort() method no longer has a "cmp" keyword argument. This class acts as an adapter which converts a cmp function @@ -986,7 +963,7 @@ class cmp_sort_key(object): def __call__(self, lhs): return self._cmp_key(self._cmp_func, lhs) - class _cmp_key(object): + class _cmp_key: __slots__ = ("_cmp_func", "_obj") def __init__(self, cmp_func, obj): @@ -1057,18 +1034,16 @@ def _do_stat(filename, follow_links=True): try: if follow_links: return os.stat(filename) - else: - return os.lstat(filename) + return os.lstat(filename) except OSError as oe: func_call = "stat('%s')" % filename if oe.errno == errno.EPERM: raise OperationNotPermitted(func_call) - elif oe.errno == errno.EACCES: + if oe.errno == errno.EACCES: raise PermissionDenied(func_call) - elif oe.errno == errno.ENOENT: + if oe.errno == errno.ENOENT: raise FileNotFound(filename) - else: - raise + raise def apply_permissions(filename, uid=-1, gid=-1, mode=-1, mask=-1, stat_cached=None, follow_links=True): @@ -1316,29 +1291,10 @@ class atomic_ofstream(ObjectProxy): def _get_target(self): return object.__getattribute__(self, '_file') - if sys.hexversion >= 0x3000000: - - def __getattribute__(self, attr): - if attr in ('close', 'abort', '__del__'): - return object.__getattribute__(self, attr) - return getattr(object.__getattribute__(self, '_file'), attr) - - else: - - # For TextIOWrapper, automatically coerce write calls to - # unicode, in order to avoid TypeError when writing raw - # bytes with python2. - - def __getattribute__(self, attr): - if attr in ('close', 'abort', 'write', '__del__'): - return object.__getattribute__(self, attr) - return getattr(object.__getattribute__(self, '_file'), attr) - - def write(self, s): - f = object.__getattribute__(self, '_file') - if isinstance(f, io.TextIOWrapper): - s = _unicode_decode(s) - return f.write(s) + def __getattribute__(self, attr): + if attr in ('close', 'abort', '__del__'): + return object.__getattribute__(self, attr) + return getattr(object.__getattribute__(self, '_file'), attr) def close(self): """Closes the temporary file, copies permissions (if possible), @@ -1519,8 +1475,7 @@ class LazyItemsDict(UserDict): self[item_key] = result return result - else: - return UserDict.__getitem__(self, item_key) + return UserDict.__getitem__(self, item_key) def __setitem__(self, item_key, value): if item_key in self.lazy_items: @@ -1564,7 +1519,7 @@ class LazyItemsDict(UserDict): UserDict.__setitem__(result, k_copy, deepcopy(self[k], memo)) return result - class _LazyItem(object): + class _LazyItem: __slots__ = ('func', 'pargs', 'kwargs', 'singleton') @@ -1599,7 +1554,7 @@ class LazyItemsDict(UserDict): result.singleton = deepcopy(self.singleton, memo) return result -class ConfigProtect(object): +class ConfigProtect: def __init__(self, myroot, protect_list, mask_list, case_insensitive=False): self.myroot = myroot diff --git a/lib/portage/util/_async/AsyncFunction.py b/lib/portage/util/_async/AsyncFunction.py index 1dffa36cc..db77a6f43 100644 --- a/lib/portage/util/_async/AsyncFunction.py +++ b/lib/portage/util/_async/AsyncFunction.py @@ -1,4 +1,4 @@ -# Copyright 2015 Gentoo Foundation +# Copyright 2015-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import pickle @@ -23,7 +23,7 @@ class AsyncFunction(ForkProcess): def _start(self): pr, pw = os.pipe() - self.fd_pipes = {} + self.fd_pipes = {} if self.fd_pipes is None else self.fd_pipes self.fd_pipes[pw] = pw self._async_func_reader_pw = pw self._async_func_reader = PipeReader( diff --git a/lib/portage/util/_async/BuildLogger.py b/lib/portage/util/_async/BuildLogger.py new file mode 100644 index 000000000..f5fea77ea --- /dev/null +++ b/lib/portage/util/_async/BuildLogger.py @@ -0,0 +1,109 @@ +# Copyright 2020 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +import subprocess + +from portage import os +from portage.util import shlex_split +from _emerge.AsynchronousTask import AsynchronousTask +from portage.util._async.PipeLogger import PipeLogger +from portage.util._async.PopenProcess import PopenProcess +from portage.util.futures import asyncio +from portage.util.futures.compat_coroutine import coroutine + + +class BuildLogger(AsynchronousTask): + """ + Write to a log file, with compression support provided by PipeLogger. + If the log_filter_file parameter is specified, then it is interpreted + as a command to execute which filters log output (see the + PORTAGE_LOG_FILTER_FILE_CMD variable in make.conf(5)). The stdin property + provides access to a writable binary file stream (refers to a pipe) + that log content should be written to (usually redirected from + subprocess stdout and stderr streams). + """ + + __slots__ = ('env', 'log_path', 'log_filter_file', '_main_task', '_stdin') + + @property + def stdin(self): + return self._stdin + + def _start(self): + filter_proc = None + log_input = None + if self.log_path is not None: + log_filter_file = self.log_filter_file + if log_filter_file is not None: + split_value = shlex_split(log_filter_file) + log_filter_file = split_value if split_value else None + if log_filter_file: + filter_input, stdin = os.pipe() + log_input, filter_output = os.pipe() + try: + filter_proc = PopenProcess( + proc=subprocess.Popen( + log_filter_file, + env=self.env, + stdin=filter_input, + stdout=filter_output, + stderr=filter_output, + ), + scheduler=self.scheduler, + ) + filter_proc.start() + except EnvironmentError: + # Maybe the command is missing or broken somehow... + os.close(filter_input) + os.close(stdin) + os.close(log_input) + os.close(filter_output) + else: + self._stdin = os.fdopen(stdin, 'wb', 0) + os.close(filter_input) + os.close(filter_output) + + if self._stdin is None: + # Since log_filter_file is unspecified or refers to a file + # that is missing or broken somehow, create a pipe that + # logs directly to pipe_logger. + log_input, stdin = os.pipe() + self._stdin = os.fdopen(stdin, 'wb', 0) + + # Set background=True so that pipe_logger does not log to stdout. + pipe_logger = PipeLogger(background=True, + scheduler=self.scheduler, input_fd=log_input, + log_file_path=self.log_path) + pipe_logger.start() + + self._main_task = asyncio.ensure_future(self._main(filter_proc, pipe_logger), loop=self.scheduler) + self._main_task.add_done_callback(self._main_exit) + + @coroutine + def _main(self, filter_proc, pipe_logger): + try: + if pipe_logger.poll() is None: + yield pipe_logger.async_wait() + if filter_proc is not None and filter_proc.poll() is None: + yield filter_proc.async_wait() + except asyncio.CancelledError: + if pipe_logger.poll() is None: + pipe_logger.cancel() + if filter_proc is not None and filter_proc.poll() is None: + filter_proc.cancel() + raise + + def _cancel(self): + if self._main_task is not None: + self._main_task.done() or self._main_task.cancel() + if self._stdin is not None and not self._stdin.closed: + self._stdin.close() + + def _main_exit(self, main_task): + try: + main_task.result() + except asyncio.CancelledError: + self.cancel() + self._was_cancelled() + self.returncode = self.returncode or 0 + self._async_wait() diff --git a/lib/portage/util/_async/ForkProcess.py b/lib/portage/util/_async/ForkProcess.py index d84e93833..eb01a6232 100644 --- a/lib/portage/util/_async/ForkProcess.py +++ b/lib/portage/util/_async/ForkProcess.py @@ -1,37 +1,123 @@ -# Copyright 2012-2013 Gentoo Foundation +# Copyright 2012-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 +import fcntl +import functools +import multiprocessing import signal import sys -import traceback import portage from portage import os +from portage.util.futures import asyncio +from portage.util.futures.compat_coroutine import coroutine from _emerge.SpawnProcess import SpawnProcess class ForkProcess(SpawnProcess): - __slots__ = () + __slots__ = ('_proc', '_proc_join_task') + + # Number of seconds between poll attempts for process exit status + # (after the sentinel has become ready). + _proc_join_interval = 0.1 def _spawn(self, args, fd_pipes=None, **kwargs): """ - Fork a subprocess, apply local settings, and call fetch(). + Override SpawnProcess._spawn to fork a subprocess that calls + self._run(). This uses multiprocessing.Process in order to leverage + any pre-fork and post-fork interpreter housekeeping that it provides, + promoting a healthy state for the forked interpreter. """ - - parent_pid = os.getpid() - pid = None + # Since multiprocessing.Process closes sys.__stdin__, create a + # temporary duplicate of fd_pipes[0] so that sys.__stdin__ can + # be restored in the subprocess, in case this is needed for + # things like PROPERTIES=interactive support. + stdin_dup = None try: - pid = os.fork() + stdin_fd = fd_pipes.get(0) + if stdin_fd is not None and stdin_fd == portage._get_stdin().fileno(): + stdin_dup = os.dup(stdin_fd) + fcntl.fcntl(stdin_dup, fcntl.F_SETFD, + fcntl.fcntl(stdin_fd, fcntl.F_GETFD)) + fd_pipes[0] = stdin_dup + self._proc = multiprocessing.Process(target=self._bootstrap, args=(fd_pipes,)) + self._proc.start() + finally: + if stdin_dup is not None: + os.close(stdin_dup) + + self._proc_join_task = asyncio.ensure_future( + self._proc_join(self._proc)) + self._proc_join_task.add_done_callback( + functools.partial(self._proc_join_done, self._proc)) + + return [self._proc.pid] + + def _cancel(self): + if self._proc is None: + super(ForkProcess, self)._cancel() + else: + self._proc.terminate() + + def _async_wait(self): + if self._proc_join_task is None: + super(ForkProcess, self)._async_wait() - if pid != 0: - if not isinstance(pid, int): - raise AssertionError( - "fork returned non-integer: %s" % (repr(pid),)) - return [pid] + def _async_waitpid(self): + if self._proc_join_task is None: + super(ForkProcess, self)._async_waitpid() - rval = 1 + @coroutine + def _proc_join(self, proc): + sentinel_reader = self.scheduler.create_future() + self.scheduler.add_reader(proc.sentinel, + lambda: sentinel_reader.done() or sentinel_reader.set_result(None)) + try: + yield sentinel_reader + finally: + # If multiprocessing.Process supports the close method, then + # access to proc.sentinel will raise ValueError if the + # sentinel has been closed. In this case it's not safe to call + # remove_reader, since the file descriptor may have been closed + # and then reallocated to a concurrent coroutine. When the + # close method is not supported, proc.sentinel remains open + # until proc's finalizer is called. try: + self.scheduler.remove_reader(proc.sentinel) + except ValueError: + pass + + # Now that proc.sentinel is ready, poll until process exit + # status has become available. + while True: + proc.join(0) + if proc.exitcode is not None: + break + yield asyncio.sleep(self._proc_join_interval) + + def _proc_join_done(self, proc, future): + future.cancelled() or future.result() + self._was_cancelled() + if self.returncode is None: + self.returncode = proc.exitcode + self._proc = None + if hasattr(proc, 'close'): + proc.close() + self._proc_join_task = None + self._async_wait() + + def _unregister(self): + super(ForkProcess, self)._unregister() + if self._proc is not None: + if self._proc.is_alive(): + self._proc.terminate() + self._proc = None + if self._proc_join_task is not None: + self._proc_join_task.cancel() + self._proc_join_task = None + + def _bootstrap(self, fd_pipes): # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. signal.signal(signal.SIGINT, signal.SIG_DFL) @@ -52,24 +138,22 @@ class ForkProcess(SpawnProcess): # (see _setup_pipes docstring). portage.process._setup_pipes(fd_pipes, close_fds=False) - rval = self._run() - except SystemExit: - raise - except: - traceback.print_exc() - # os._exit() skips stderr flush! - sys.stderr.flush() - finally: - os._exit(rval) + # Since multiprocessing.Process closes sys.__stdin__ and + # makes sys.stdin refer to os.devnull, restore it when + # appropriate. + if 0 in fd_pipes: + # It's possible that sys.stdin.fileno() is already 0, + # and in that case the above _setup_pipes call will + # have already updated its identity via dup2. Otherwise, + # perform the dup2 call now, and also copy the file + # descriptor flags. + if sys.stdin.fileno() != 0: + os.dup2(0, sys.stdin.fileno()) + fcntl.fcntl(sys.stdin.fileno(), fcntl.F_SETFD, + fcntl.fcntl(0, fcntl.F_GETFD)) + sys.__stdin__ = sys.stdin - finally: - if pid == 0 or (pid is None and os.getpid() != parent_pid): - # Call os._exit() from a finally block in order - # to suppress any finally blocks from earlier - # in the call stack (see bug #345289). This - # finally block has to be setup before the fork - # in order to avoid a race condition. - os._exit(1) + sys.exit(self._run()) def _run(self): raise NotImplementedError(self) diff --git a/lib/portage/util/_async/PipeLogger.py b/lib/portage/util/_async/PipeLogger.py index a4258f350..060483f0b 100644 --- a/lib/portage/util/_async/PipeLogger.py +++ b/lib/portage/util/_async/PipeLogger.py @@ -1,13 +1,16 @@ -# Copyright 2008-2018 Gentoo Foundation +# Copyright 2008-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import fcntl import errno import gzip -import sys import portage from portage import os, _encodings, _unicode_encode +from portage.util.futures import asyncio +from portage.util.futures._asyncio.streams import _writer +from portage.util.futures.compat_coroutine import coroutine +from portage.util.futures.unix_events import _set_nonblocking from _emerge.AbstractPollTask import AbstractPollTask class PipeLogger(AbstractPollTask): @@ -21,13 +24,16 @@ class PipeLogger(AbstractPollTask): """ __slots__ = ("input_fd", "log_file_path", "stdout_fd") + \ - ("_log_file", "_log_file_real") + ("_io_loop_task", "_log_file", "_log_file_nb", "_log_file_real") def _start(self): log_file_path = self.log_file_path - if log_file_path is not None: - + if hasattr(log_file_path, 'write'): + self._log_file_nb = True + self._log_file = log_file_path + _set_nonblocking(self._log_file.fileno()) + elif log_file_path is not None: self._log_file = open(_unicode_encode(log_file_path, encoding=_encodings['fs'], errors='strict'), mode='ab') if log_file_path.endswith('.gz'): @@ -40,24 +46,15 @@ class PipeLogger(AbstractPollTask): mode=0o660) if isinstance(self.input_fd, int): - fd = self.input_fd - else: - fd = self.input_fd.fileno() + self.input_fd = os.fdopen(self.input_fd, 'rb', 0) + + fd = self.input_fd.fileno() fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK) - # FD_CLOEXEC is enabled by default in Python >=3.4. - if sys.hexversion < 0x3040000: - try: - fcntl.FD_CLOEXEC - except AttributeError: - pass - else: - fcntl.fcntl(fd, fcntl.F_SETFD, - fcntl.fcntl(fd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) - - self.scheduler.add_reader(fd, self._output_handler, fd) + self._io_loop_task = asyncio.ensure_future(self._io_loop(self.input_fd), loop=self.scheduler) + self._io_loop_task.add_done_callback(self._io_loop_done) self._registered = True def _cancel(self): @@ -65,80 +62,113 @@ class PipeLogger(AbstractPollTask): if self.returncode is None: self.returncode = self._cancelled_returncode - def _output_handler(self, fd): - + @coroutine + def _io_loop(self, input_file): background = self.background stdout_fd = self.stdout_fd log_file = self._log_file + fd = input_file.fileno() while True: buf = self._read_buf(fd) if buf is None: # not a POLLIN event, EAGAIN, etc... - break + future = self.scheduler.create_future() + self.scheduler.add_reader(fd, future.set_result, None) + try: + yield future + finally: + # The loop and input file may have been closed. + if not self.scheduler.is_closed(): + future.done() or future.cancel() + # Do not call remove_reader in cases where fd has + # been closed and then re-allocated to a concurrent + # coroutine as in bug 716636. + if not input_file.closed: + self.scheduler.remove_reader(fd) + continue if not buf: # EOF - self._unregister() - self.returncode = self.returncode or os.EX_OK - self._async_wait() - break - - else: - if not background and stdout_fd is not None: - failures = 0 - stdout_buf = buf - while stdout_buf: - try: - stdout_buf = \ - stdout_buf[os.write(stdout_fd, stdout_buf):] - except OSError as e: - if e.errno != errno.EAGAIN: - raise - del e - failures += 1 - if failures > 50: - # Avoid a potentially infinite loop. In - # most cases, the failure count is zero - # and it's unlikely to exceed 1. - raise - - # This means that a subprocess has put an inherited - # stdio file descriptor (typically stdin) into - # O_NONBLOCK mode. This is not acceptable (see bug - # #264435), so revert it. We need to use a loop - # here since there's a race condition due to - # parallel processes being able to change the - # flags on the inherited file descriptor. - # TODO: When possible, avoid having child processes - # inherit stdio file descriptors from portage - # (maybe it can't be avoided with - # PROPERTIES=interactive). - fcntl.fcntl(stdout_fd, fcntl.F_SETFL, - fcntl.fcntl(stdout_fd, - fcntl.F_GETFL) ^ os.O_NONBLOCK) - - if log_file is not None: + return + + if not background and stdout_fd is not None: + failures = 0 + stdout_buf = buf + while stdout_buf: + try: + stdout_buf = \ + stdout_buf[os.write(stdout_fd, stdout_buf):] + except OSError as e: + if e.errno != errno.EAGAIN: + raise + del e + failures += 1 + if failures > 50: + # Avoid a potentially infinite loop. In + # most cases, the failure count is zero + # and it's unlikely to exceed 1. + raise + + # This means that a subprocess has put an inherited + # stdio file descriptor (typically stdin) into + # O_NONBLOCK mode. This is not acceptable (see bug + # #264435), so revert it. We need to use a loop + # here since there's a race condition due to + # parallel processes being able to change the + # flags on the inherited file descriptor. + # TODO: When possible, avoid having child processes + # inherit stdio file descriptors from portage + # (maybe it can't be avoided with + # PROPERTIES=interactive). + fcntl.fcntl(stdout_fd, fcntl.F_SETFL, + fcntl.fcntl(stdout_fd, + fcntl.F_GETFL) ^ os.O_NONBLOCK) + + if log_file is not None: + if self._log_file_nb: + # Use the _writer function which uses os.write, since the + # log_file.write method looses data when an EAGAIN occurs. + yield _writer(log_file, buf, loop=self.scheduler) + else: + # For gzip.GzipFile instances, the above _writer function + # will not work because data written directly to the file + # descriptor bypasses compression. log_file.write(buf) log_file.flush() + def _io_loop_done(self, future): + try: + future.result() + except asyncio.CancelledError: + self.cancel() + self._was_cancelled() + self.returncode = self.returncode or os.EX_OK + self._async_wait() + def _unregister(self): if self.input_fd is not None: if isinstance(self.input_fd, int): - self.scheduler.remove_reader(self.input_fd) os.close(self.input_fd) - else: + elif not self.input_fd.closed: self.scheduler.remove_reader(self.input_fd.fileno()) self.input_fd.close() self.input_fd = None + if self._io_loop_task is not None: + if not self.scheduler.is_closed(): + self._io_loop_task.done() or self._io_loop_task.cancel() + self._io_loop_task = None + if self.stdout_fd is not None: os.close(self.stdout_fd) self.stdout_fd = None if self._log_file is not None: - self._log_file.close() + if not self._log_file.closed: + self.scheduler.remove_writer(self._log_file.fileno()) + self._log_file.close() self._log_file = None if self._log_file_real is not None: diff --git a/lib/portage/util/_async/SchedulerInterface.py b/lib/portage/util/_async/SchedulerInterface.py index ec6417da1..3ff250d1d 100644 --- a/lib/portage/util/_async/SchedulerInterface.py +++ b/lib/portage/util/_async/SchedulerInterface.py @@ -1,4 +1,4 @@ -# Copyright 2012-2018 Gentoo Foundation +# Copyright 2012-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import gzip @@ -7,6 +7,8 @@ import errno from portage import _encodings from portage import _unicode_encode from portage.util import writemsg_level +from portage.util.futures._asyncio.streams import _writer +from portage.util.futures.compat_coroutine import coroutine from ..SlotObject import SlotObject class SchedulerInterface(SlotObject): @@ -53,6 +55,34 @@ class SchedulerInterface(SlotObject): def _return_false(): return False + @coroutine + def async_output(self, msg, log_file=None, background=None, + level=0, noiselevel=-1): + """ + Output a msg to stdio (if not in background) and to a log file + if provided. + + @param msg: a message string, including newline if appropriate + @type msg: str + @param log_file: log file in binary mode + @type log_file: file + @param background: send messages only to log (not to stdio) + @type background: bool + @param level: a numeric logging level (see the logging module) + @type level: int + @param noiselevel: passed directly to writemsg + @type noiselevel: int + """ + global_background = self._is_background() + if background is None or global_background: + background = global_background + + if not background: + writemsg_level(msg, level=level, noiselevel=noiselevel) + + if log_file is not None: + yield _writer(log_file, _unicode_encode(msg)) + def output(self, msg, log_path=None, background=None, level=0, noiselevel=-1): """ diff --git a/lib/portage/util/_compare_files.py b/lib/portage/util/_compare_files.py index bd993e501..de97a9d9d 100644 --- a/lib/portage/util/_compare_files.py +++ b/lib/portage/util/_compare_files.py @@ -1,4 +1,4 @@ -# Copyright 2019 Gentoo Authors +# Copyright 2019-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 __all__ = ["compare_files"] @@ -6,7 +6,6 @@ __all__ = ["compare_files"] import io import os import stat -import sys from portage import _encodings from portage import _unicode_encode @@ -49,20 +48,12 @@ def compare_files(file1, file2, skipped_types=()): if "xattr" not in skipped_types and sorted(xattr.get_all(file1, nofollow=True)) != sorted(xattr.get_all(file2, nofollow=True)): differences.append("xattr") - if sys.hexversion >= 0x3030000: - if "atime" not in skipped_types and file1_stat.st_atime_ns != file2_stat.st_atime_ns: - differences.append("atime") - if "mtime" not in skipped_types and file1_stat.st_mtime_ns != file2_stat.st_mtime_ns: - differences.append("mtime") - if "ctime" not in skipped_types and file1_stat.st_ctime_ns != file2_stat.st_ctime_ns: - differences.append("ctime") - else: - if "atime" not in skipped_types and file1_stat.st_atime != file2_stat.st_atime: - differences.append("atime") - if "mtime" not in skipped_types and file1_stat.st_mtime != file2_stat.st_mtime: - differences.append("mtime") - if "ctime" not in skipped_types and file1_stat.st_ctime != file2_stat.st_ctime: - differences.append("ctime") + if "atime" not in skipped_types and file1_stat.st_atime_ns != file2_stat.st_atime_ns: + differences.append("atime") + if "mtime" not in skipped_types and file1_stat.st_mtime_ns != file2_stat.st_mtime_ns: + differences.append("mtime") + if "ctime" not in skipped_types and file1_stat.st_ctime_ns != file2_stat.st_ctime_ns: + differences.append("ctime") if "type" in differences: pass diff --git a/lib/portage/util/_desktop_entry.py b/lib/portage/util/_desktop_entry.py index ee6572588..74053a30f 100644 --- a/lib/portage/util/_desktop_entry.py +++ b/lib/portage/util/_desktop_entry.py @@ -1,7 +1,6 @@ # Copyright 2012-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -import io import re import subprocess import sys @@ -25,7 +24,12 @@ def parse_desktop_entry(path): return parser -_trivial_warnings = re.compile(r' looks redundant with value ') +_trivial_warnings = re.compile(r' looks ' + # >=desktop-file-utils-0.25 + r'(?:the same as that of key|' + + # <desktop-file-utils-0.25 + r'redundant with value) ') _ignored_errors = ( # Ignore error for emacs.desktop: diff --git a/lib/portage/util/_dyn_libs/LinkageMapELF.py b/lib/portage/util/_dyn_libs/LinkageMapELF.py index 473a1243d..954a956c6 100644 --- a/lib/portage/util/_dyn_libs/LinkageMapELF.py +++ b/lib/portage/util/_dyn_libs/LinkageMapELF.py @@ -6,7 +6,6 @@ import errno import itertools import logging import subprocess -import sys import portage from portage import _encodings @@ -27,10 +26,6 @@ from portage.util import writemsg_level from portage.util._dyn_libs.NeededEntry import NeededEntry from portage.util.elf.header import ELFHeader -if sys.hexversion >= 0x3000000: - _unicode = str -else: - _unicode = unicode # Map ELF e_machine values from NEEDED.ELF.2 to approximate multilib # categories. This approximation will produce incorrect results on x32 @@ -55,7 +50,7 @@ _approx_multilib_categories = { "X86_64": "x86_64", } -class LinkageMapELF(object): +class LinkageMapELF: """Models dynamic linker dependencies.""" @@ -63,7 +58,7 @@ class LinkageMapELF(object): _soname_map_class = slot_dict_class( ("consumers", "providers"), prefix="") - class _obj_properties_class(object): + class _obj_properties_class: __slots__ = ("arch", "needed", "runpaths", "soname", "alt_paths", "owner",) @@ -106,7 +101,7 @@ class LinkageMapELF(object): self._obj_key_cache[path] = key return key - class _ObjectKey(object): + class _ObjectKey: """Helper class used as _obj_properties keys for objects.""" @@ -333,7 +328,7 @@ class LinkageMapELF(object): entry.multilib_category = compute_multilib_category(elf_header) entry.filename = entry.filename[root_len:] owner = plibs.pop(entry.filename, None) - lines.append((owner, "scanelf", _unicode(entry))) + lines.append((owner, "scanelf", str(entry))) proc.wait() proc.stdout.close() @@ -489,7 +484,7 @@ class LinkageMapELF(object): os = _os_merge - class _LibraryCache(object): + class _LibraryCache: """ Caches properties associated with paths. @@ -520,24 +515,23 @@ class LinkageMapELF(object): """ if obj in cache_self.cache: return cache_self.cache[obj] - else: - obj_key = self._obj_key(obj) - # Check that the library exists on the filesystem. - if obj_key.file_exists(): - # Get the arch and soname from LinkageMap._obj_properties if - # it exists. Otherwise, None. - obj_props = self._obj_properties.get(obj_key) - if obj_props is None: - arch = None - soname = None - else: - arch = obj_props.arch - soname = obj_props.soname - return cache_self.cache.setdefault(obj, \ - (arch, soname, obj_key, True)) + + obj_key = self._obj_key(obj) + # Check that the library exists on the filesystem. + if obj_key.file_exists(): + # Get the arch and soname from LinkageMap._obj_properties if + # it exists. Otherwise, None. + obj_props = self._obj_properties.get(obj_key) + if obj_props is None: + arch = None + soname = None else: - return cache_self.cache.setdefault(obj, \ - (None, None, obj_key, False)) + arch = obj_props.arch + soname = obj_props.soname + return cache_self.cache.setdefault(obj, \ + (arch, soname, obj_key, True)) + return cache_self.cache.setdefault(obj, \ + (None, None, obj_key, False)) rValue = {} cache = _LibraryCache() diff --git a/lib/portage/util/_dyn_libs/NeededEntry.py b/lib/portage/util/_dyn_libs/NeededEntry.py index 70ff99100..20dc2f779 100644 --- a/lib/portage/util/_dyn_libs/NeededEntry.py +++ b/lib/portage/util/_dyn_libs/NeededEntry.py @@ -1,15 +1,12 @@ # Copyright 2015 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - -import sys from portage import _encodings, _unicode_encode from portage.exception import InvalidData from portage.localization import _ -class NeededEntry(object): +class NeededEntry: """ Represents one entry (line) from a NEEDED.ELF.2 file. The entry must have 5 or more semicolon-delimited fields in order to be @@ -75,13 +72,3 @@ class NeededEntry(object): (self.multilib_category if self.multilib_category is not None else "") ]) + "\n" - - if sys.hexversion < 0x3000000: - - __unicode__ = __str__ - - def __str__(self): - return _unicode_encode(self.__unicode__(), - encoding=_encodings['content']) - - __str__.__doc__ = __unicode__.__doc__ diff --git a/lib/portage/util/_dyn_libs/PreservedLibsRegistry.py b/lib/portage/util/_dyn_libs/PreservedLibsRegistry.py index f83b82a31..d6f1d5e29 100644 --- a/lib/portage/util/_dyn_libs/PreservedLibsRegistry.py +++ b/lib/portage/util/_dyn_libs/PreservedLibsRegistry.py @@ -1,16 +1,11 @@ -# Copyright 1998-2014 Gentoo Foundation +# Copyright 1998-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import errno import json import logging +import pickle import stat -import sys - -try: - import cPickle as pickle -except ImportError: - import pickle from portage import abssymlink from portage import os @@ -25,11 +20,8 @@ from portage.util import writemsg_level from portage.versions import cpv_getkey from portage.locks import lockfile, unlockfile -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str -class PreservedLibsRegistry(object): +class PreservedLibsRegistry: """ This class handles the tracking of preserved library objects """ # JSON read support has been available since portage-2.2.0_alpha89. @@ -38,11 +30,8 @@ class PreservedLibsRegistry(object): _json_write_opts = { "ensure_ascii": False, "indent": "\t", - "sort_keys": True + "sort_keys": True, } - if sys.hexversion < 0x30200F0: - # indent only supports int number of spaces - _json_write_opts["indent"] = 4 def __init__(self, root, filename): """ @@ -154,7 +143,7 @@ class PreservedLibsRegistry(object): int conversion and a possible ValueError resulting from vardb corruption. """ - if not isinstance(counter, basestring): + if not isinstance(counter, str): counter = str(counter) return _unicode_decode(counter).strip() diff --git a/lib/portage/util/_dyn_libs/soname_deps.py b/lib/portage/util/_dyn_libs/soname_deps.py index 544cbc8f1..9010abb27 100644 --- a/lib/portage/util/_dyn_libs/soname_deps.py +++ b/lib/portage/util/_dyn_libs/soname_deps.py @@ -1,8 +1,6 @@ # Copyright 2015 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - import collections import fnmatch import functools @@ -17,7 +15,7 @@ from portage.util import ( ) -class SonameDepsProcessor(object): +class SonameDepsProcessor: """ Processes NEEDED.ELF.2 entries for one package, in order to generate REQUIRES and PROVIDES metadata. diff --git a/lib/portage/util/_eventloop/EventLoop.py b/lib/portage/util/_eventloop/EventLoop.py index ffd12cff9..b111238a6 100644 --- a/lib/portage/util/_eventloop/EventLoop.py +++ b/lib/portage/util/_eventloop/EventLoop.py @@ -1,4 +1,4 @@ -# Copyright 1999-2018 Gentoo Foundation +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from __future__ import division @@ -10,13 +10,10 @@ import logging import os import select import signal -import sys +import time import traceback -try: - import asyncio as _real_asyncio -except ImportError: - _real_asyncio = None +import asyncio as _real_asyncio try: import fcntl @@ -37,12 +34,11 @@ portage.proxy.lazyimport.lazyimport(globals(), ) from portage.util import writemsg_level -from portage.util.monotonic import monotonic from ..SlotObject import SlotObject from .PollConstants import PollConstants from .PollSelectAdapter import PollSelectAdapter -class EventLoop(object): +class EventLoop: """ An event loop, intended to be compatible with the GLib event loop. Call the iteration method in order to execute one iteration of the @@ -69,7 +65,7 @@ class EventLoop(object): __slots__ = ("args", "function", "calling", "interval", "source_id", "timestamp") - class _handle(object): + class _handle: """ A callback wrapper object, compatible with asyncio.Handle. """ @@ -86,7 +82,7 @@ class EventLoop(object): """ self._loop.source_remove(self._callback_id) - class _call_soon_callback(object): + class _call_soon_callback: """ Wraps a call_soon callback, and always returns False, since these callbacks are only supposed to run once. @@ -101,7 +97,7 @@ class EventLoop(object): self._callback(*self._args) return False - class _selector_callback(object): + class _selector_callback: """ Wraps an callback, and always returns True, for callbacks that are supposed to run repeatedly. @@ -168,18 +164,6 @@ class EventLoop(object): # IOError: [Errno 38] Function not implemented pass else: - - # FD_CLOEXEC is enabled by default in Python >=3.4. - if sys.hexversion < 0x3040000 and fcntl is not None: - try: - fcntl.FD_CLOEXEC - except AttributeError: - pass - else: - fcntl.fcntl(epoll_obj.fileno(), fcntl.F_SETFD, - fcntl.fcntl(epoll_obj.fileno(), - fcntl.F_GETFD) | fcntl.FD_CLOEXEC) - self._poll_obj = _epoll_adapter(epoll_obj) self.IO_ERR = select.EPOLLERR self.IO_HUP = select.EPOLLHUP @@ -432,17 +416,6 @@ class EventLoop(object): fcntl.fcntl(self._sigchld_read, fcntl.F_GETFL) | os.O_NONBLOCK) - # FD_CLOEXEC is enabled by default in Python >=3.4. - if sys.hexversion < 0x3040000: - try: - fcntl.FD_CLOEXEC - except AttributeError: - pass - else: - fcntl.fcntl(self._sigchld_read, fcntl.F_SETFD, - fcntl.fcntl(self._sigchld_read, - fcntl.F_GETFD) | fcntl.FD_CLOEXEC) - # The IO watch is dynamically registered and unregistered as # needed, since we don't want to consider it as a valid source # of events when there are no child listeners. It's important @@ -887,7 +860,7 @@ class EventLoop(object): epoch, precision, accuracy and drift are unspecified and may differ per event loop. """ - return monotonic() + return time.monotonic() def call_later(self, delay, callback, *args, **kwargs): """ @@ -985,9 +958,8 @@ class EventLoop(object): executor = ForkExecutor(loop=self) self._default_executor = executor future = executor.submit(func, *args) - if _real_asyncio is not None: - future = _real_asyncio.wrap_future(future, - loop=self._asyncio_wrapper) + future = _real_asyncio.wrap_future(future, + loop=self._asyncio_wrapper) return future def is_running(self): @@ -1148,7 +1120,7 @@ def create_poll_instance(): return select.poll() return PollSelectAdapter() -class _epoll_adapter(object): +class _epoll_adapter: """ Wraps a select.epoll instance in order to make it compatible with select.poll instances. This is necessary since epoll instances diff --git a/lib/portage/util/_eventloop/PollConstants.py b/lib/portage/util/_eventloop/PollConstants.py index d0270a996..c5700d108 100644 --- a/lib/portage/util/_eventloop/PollConstants.py +++ b/lib/portage/util/_eventloop/PollConstants.py @@ -2,7 +2,7 @@ # Distributed under the terms of the GNU General Public License v2 import select -class PollConstants(object): +class PollConstants: """ Provides POLL* constants that are equivalent to those from the @@ -15,4 +15,3 @@ class PollConstants(object): locals()[k] = getattr(select, k, v) v *= 2 del k, v - diff --git a/lib/portage/util/_eventloop/PollSelectAdapter.py b/lib/portage/util/_eventloop/PollSelectAdapter.py index 32b404b67..08dc664a4 100644 --- a/lib/portage/util/_eventloop/PollSelectAdapter.py +++ b/lib/portage/util/_eventloop/PollSelectAdapter.py @@ -6,7 +6,7 @@ from __future__ import division from .PollConstants import PollConstants import select -class PollSelectAdapter(object): +class PollSelectAdapter: """ Use select to emulate a poll object, for @@ -73,4 +73,3 @@ class PollSelectAdapter(object): for fd in select_events[0]: poll_events.append((fd, PollConstants.POLLIN)) return poll_events - diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py b/lib/portage/util/_eventloop/asyncio_event_loop.py index ce7e06923..605e7243b 100644 --- a/lib/portage/util/_eventloop/asyncio_event_loop.py +++ b/lib/portage/util/_eventloop/asyncio_event_loop.py @@ -4,13 +4,8 @@ import os import signal -try: - import asyncio as _real_asyncio - from asyncio.events import AbstractEventLoop as _AbstractEventLoop -except ImportError: - # Allow ImportModulesTestCase to succeed. - _real_asyncio = None - _AbstractEventLoop = object +import asyncio as _real_asyncio +from asyncio.events import AbstractEventLoop as _AbstractEventLoop import portage diff --git a/lib/portage/util/_eventloop/global_event_loop.py b/lib/portage/util/_eventloop/global_event_loop.py index 2f6371dc1..1db958d2e 100644 --- a/lib/portage/util/_eventloop/global_event_loop.py +++ b/lib/portage/util/_eventloop/global_event_loop.py @@ -1,22 +1,16 @@ -# Copyright 2012 Gentoo Foundation +# Copyright 2012-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import os -import sys from .EventLoop import EventLoop from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop -_asyncio_enabled = sys.version_info >= (3, 4) -_default_constructor = AsyncioEventLoop if _asyncio_enabled else EventLoop - -# If _default_constructor doesn't support multiprocessing, -# then _multiprocessing_constructor is used in subprocesses. -_multiprocessing_constructor = EventLoop _MAIN_PID = os.getpid() _instances = {} + def global_event_loop(): """ Get a global EventLoop (or compatible object) instance which @@ -28,9 +22,11 @@ def global_event_loop(): if instance is not None: return instance - constructor = _default_constructor + constructor = AsyncioEventLoop + # If the default constructor doesn't support multiprocessing, + # then multiprocessing constructor is used in subprocesses. if not constructor.supports_multiprocessing and pid != _MAIN_PID: - constructor = _multiprocessing_constructor + constructor = EventLoop # Use the _asyncio_wrapper attribute, so that unit tests can compare # the reference to one retured from _wrap_loop(), since they should diff --git a/lib/portage/util/_urlopen.py b/lib/portage/util/_urlopen.py index 1d8ba3fd3..b46d1554c 100644 --- a/lib/portage/util/_urlopen.py +++ b/lib/portage/util/_urlopen.py @@ -1,24 +1,14 @@ -# Copyright 2012-2019 Gentoo Authors +# Copyright 2012-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import io -import sys from datetime import datetime from time import mktime from email.utils import formatdate, parsedate +from urllib.request import urlopen as _urlopen +import urllib.parse as urllib_parse +import urllib.request as urllib_request -try: - from urllib.request import urlopen as _urlopen - import urllib.parse as urllib_parse - import urllib.request as urllib_request -except ImportError: - from urllib import urlopen as _urlopen - import urlparse as urllib_parse - import urllib2 as urllib_request - -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - long = int # to account for the difference between TIMESTAMP of the index' contents # and the file-'mtime' @@ -40,37 +30,36 @@ def urlopen(url, if_modified_since=None): parse_result = urllib_parse.urlparse(url) if parse_result.scheme not in ("http", "https"): return _urlopen(url) - else: - netloc = parse_result.netloc.rpartition('@')[-1] - url = urllib_parse.urlunparse((parse_result.scheme, netloc, parse_result.path, parse_result.params, parse_result.query, parse_result.fragment)) - password_manager = urllib_request.HTTPPasswordMgrWithDefaultRealm() - request = urllib_request.Request(url) - request.add_header('User-Agent', 'Gentoo Portage') - if if_modified_since: - request.add_header('If-Modified-Since', _timestamp_to_http(if_modified_since)) - if parse_result.username is not None: - password_manager.add_password(None, url, parse_result.username, parse_result.password) - auth_handler = CompressedResponseProcessor(password_manager) - opener = urllib_request.build_opener(auth_handler) - hdl = opener.open(request) - if hdl.headers.get('last-modified', ''): - try: - add_header = hdl.headers.add_header - except AttributeError: - # Python 2 - add_header = hdl.headers.addheader - add_header('timestamp', _http_to_timestamp(hdl.headers.get('last-modified'))) - return hdl + + netloc = parse_result.netloc.rpartition('@')[-1] + url = urllib_parse.urlunparse((parse_result.scheme, netloc, parse_result.path, parse_result.params, parse_result.query, parse_result.fragment)) + password_manager = urllib_request.HTTPPasswordMgrWithDefaultRealm() + request = urllib_request.Request(url) + request.add_header('User-Agent', 'Gentoo Portage') + if if_modified_since: + request.add_header('If-Modified-Since', _timestamp_to_http(if_modified_since)) + if parse_result.username is not None: + password_manager.add_password(None, url, parse_result.username, parse_result.password) + auth_handler = CompressedResponseProcessor(password_manager) + opener = urllib_request.build_opener(auth_handler) + hdl = opener.open(request) + if hdl.headers.get('last-modified', ''): + try: + add_header = hdl.headers.add_header + except AttributeError: + # Python 2 + add_header = hdl.headers.addheader + add_header('timestamp', _http_to_timestamp(hdl.headers.get('last-modified'))) + return hdl def _timestamp_to_http(timestamp): - dt = datetime.fromtimestamp(float(long(timestamp)+TIMESTAMP_TOLERANCE)) + dt = datetime.fromtimestamp(float(int(timestamp)+TIMESTAMP_TOLERANCE)) stamp = mktime(dt.timetuple()) return formatdate(timeval=stamp, localtime=False, usegmt=True) def _http_to_timestamp(http_datetime_string): - tuple = parsedate(http_datetime_string) - timestamp = mktime(tuple) - return str(long(timestamp)) + timestamp = mktime(parsedate(http_datetime_string)) + return str(int(timestamp)) class CompressedResponseProcessor(urllib_request.HTTPBasicAuthHandler): # Handler for compressed responses. diff --git a/lib/portage/util/_xattr.py b/lib/portage/util/_xattr.py index c118589a3..531c61efb 100644 --- a/lib/portage/util/_xattr.py +++ b/lib/portage/util/_xattr.py @@ -20,7 +20,7 @@ import subprocess from portage.exception import OperationNotSupported -class _XattrGetAll(object): +class _XattrGetAll: """Implement get_all() using list()/get() if there is no easy bulk method""" @classmethod diff --git a/lib/portage/util/backoff.py b/lib/portage/util/backoff.py index ee39007ef..73f69d6db 100644 --- a/lib/portage/util/backoff.py +++ b/lib/portage/util/backoff.py @@ -10,7 +10,7 @@ import random import sys -class ExponentialBackoff(object): +class ExponentialBackoff: """ An object that when called with number of previous tries, calculates an exponential delay for the next try. diff --git a/lib/portage/util/changelog.py b/lib/portage/util/changelog.py index 9fc5ab6df..362cf7717 100644 --- a/lib/portage/util/changelog.py +++ b/lib/portage/util/changelog.py @@ -1,21 +1,21 @@ #!/usr/bin/python -b -# Copyright 2009-2015 Gentoo Foundation +# Copyright 2009-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from portage.manifest import guessManifestFileType -from portage.versions import _unicode, pkgsplit, vercmp +from portage.versions import pkgsplit, vercmp -class ChangeLogTypeSort(_unicode): +class ChangeLogTypeSort(str): """ Helps to sort file names by file type and other criteria. """ def __new__(cls, status_change, file_name): - return _unicode.__new__(cls, status_change + file_name) + return str.__new__(cls, status_change + file_name) def __init__(self, status_change, file_name): - _unicode.__init__(status_change + file_name) + str.__init__(status_change + file_name) self.status_change = status_change self.file_name = file_name self.file_type = guessManifestFileType(file_name) @@ -32,16 +32,15 @@ class ChangeLogTypeSort(_unicode): if first == "EBUILD": return True - elif first == "MISC": + if first == "MISC": return second in ("EBUILD",) - elif first == "AUX": + if first == "AUX": return second in ("EBUILD", "MISC") - elif first == "DIST": + if first == "DIST": return second in ("EBUILD", "MISC", "AUX") - elif first is None: + if first is None: return False - else: - raise ValueError("Unknown file type '%s'" % first) + raise ValueError("Unknown file type '%s'" % first) def __lt__(self, other): """ @@ -55,7 +54,7 @@ class ChangeLogTypeSort(_unicode): # Sort by file type as defined by _file_type_lt(). if self._file_type_lt(self, other): return True - elif self._file_type_lt(other, self): + if self._file_type_lt(other, self): return False # Files have the same type. @@ -64,6 +63,6 @@ class ChangeLogTypeSort(_unicode): ver = "-".join(pkgsplit(self.file_name[:-7])[1:3]) other_ver = "-".join(pkgsplit(other.file_name[:-7])[1:3]) return vercmp(ver, other_ver) < 0 - else: - # Sort lexicographically. - return self.file_name < other.file_name + + # Sort lexicographically. + return self.file_name < other.file_name diff --git a/lib/portage/util/compression_probe.py b/lib/portage/util/compression_probe.py index 7d595670b..6c2140794 100644 --- a/lib/portage/util/compression_probe.py +++ b/lib/portage/util/compression_probe.py @@ -4,10 +4,7 @@ import ctypes import errno import re -import sys -if sys.hexversion >= 0x3000000: - basestring = str from portage import _encodings, _unicode_encode from portage.exception import FileNotFound, PermissionDenied @@ -87,7 +84,7 @@ def compression_probe(f): @rtype str or None """ - open_file = isinstance(f, basestring) + open_file = isinstance(f, str) if open_file: try: f = open(_unicode_encode(f, diff --git a/lib/portage/util/configparser.py b/lib/portage/util/configparser.py index c4c92a603..8563d5c8c 100644 --- a/lib/portage/util/configparser.py +++ b/lib/portage/util/configparser.py @@ -1,4 +1,4 @@ -# Copyright 2016 Gentoo Foundation +# Copyright 2016-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 __all__ = ['ConfigParserError', 'NoOptionError', 'ParsingError', @@ -9,28 +9,15 @@ __all__ = ['ConfigParserError', 'NoOptionError', 'ParsingError', # - RawConfigParser that provides no interpolation for values. import io -import sys -try: - from configparser import (Error as ConfigParserError, - NoOptionError, ParsingError, RawConfigParser) - if sys.hexversion >= 0x3020000: - from configparser import ConfigParser as SafeConfigParser - else: - from configparser import SafeConfigParser -except ImportError: - from ConfigParser import (Error as ConfigParserError, - NoOptionError, ParsingError, RawConfigParser, SafeConfigParser) +from configparser import (Error as ConfigParserError, + NoOptionError, ParsingError, RawConfigParser) +from configparser import ConfigParser as SafeConfigParser from portage import _encodings from portage import _unicode_encode -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str - - def read_configs(parser, paths): """ Read configuration files from given paths into the specified @@ -50,7 +37,7 @@ def read_configs(parser, paths): source_kwarg = 'filename' for p in paths: - if isinstance(p, basestring): + if isinstance(p, str): f = None try: f = io.open(_unicode_encode(p, diff --git a/lib/portage/util/digraph.py b/lib/portage/util/digraph.py index d279b7867..c262cddee 100644 --- a/lib/portage/util/digraph.py +++ b/lib/portage/util/digraph.py @@ -1,17 +1,14 @@ # Copyright 2010-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - __all__ = ['digraph'] import bisect from collections import deque -import sys from portage.util import writemsg -class digraph(object): +class digraph: """ A directed graph object. """ @@ -385,6 +382,3 @@ class digraph(object): __contains__ = contains empty = is_empty copy = clone - - if sys.hexversion < 0x3000000: - __nonzero__ = __bool__ diff --git a/lib/portage/util/elf/header.py b/lib/portage/util/elf/header.py index 3d2307402..bedb19eab 100644 --- a/lib/portage/util/elf/header.py +++ b/lib/portage/util/elf/header.py @@ -6,7 +6,7 @@ from portage.util.endian.decode import (decode_uint16_le, from portage.util.elf.constants import (E_ENTRY, E_MACHINE, E_TYPE, EI_CLASS, ELFCLASS32, ELFCLASS64, ELFDATA2LSB, ELFDATA2MSB) -class ELFHeader(object): +class ELFHeader: __slots__ = ('e_flags', 'e_machine', 'e_type', 'ei_class', 'ei_data') diff --git a/lib/portage/util/env_update.py b/lib/portage/util/env_update.py index a69114d80..59f5bb9cc 100644 --- a/lib/portage/util/env_update.py +++ b/lib/portage/util/env_update.py @@ -1,4 +1,4 @@ -# Copyright 2010-2014 Gentoo Foundation +# Copyright 2010-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 __all__ = ['env_update'] @@ -7,7 +7,6 @@ import errno import glob import io import stat -import sys import time import portage @@ -23,9 +22,6 @@ from portage.util.listdir import listdir from portage.dbapi.vartree import vartree from portage.package.ebuild.config import config -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - long = int def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None, env=None, writemsg_level=None, vardbapi=None): @@ -257,7 +253,7 @@ def _env_update(makelinks, target_root, prev_mtimes, contents, env, if e.errno != errno.ENOENT: raise - current_time = long(time.time()) + current_time = int(time.time()) mtime_changed = False lib_dirs = set() diff --git a/lib/portage/util/formatter.py b/lib/portage/util/formatter.py index ce6799e3f..8180a662c 100644 --- a/lib/portage/util/formatter.py +++ b/lib/portage/util/formatter.py @@ -7,7 +7,7 @@ import sys -class AbstractFormatter(object): +class AbstractFormatter: """The standard formatter.""" def __init__(self, writer): @@ -35,7 +35,7 @@ class AbstractFormatter(object): self.writer.new_styles(tuple(self.style_stack)) -class NullWriter(object): +class NullWriter: """Minimal writer interface to use in testing & inheritance. A writer which only provides the interface definition; no actions are @@ -66,4 +66,3 @@ class DumbWriter(NullWriter): def send_literal_data(self, data): self.file.write(data) - diff --git a/lib/portage/util/futures/_asyncio/__init__.py b/lib/portage/util/futures/_asyncio/__init__.py index f4b03891f..a902ad895 100644 --- a/lib/portage/util/futures/_asyncio/__init__.py +++ b/lib/portage/util/futures/_asyncio/__init__.py @@ -23,10 +23,7 @@ __all__ = ( import subprocess import sys -try: - import asyncio as _real_asyncio -except ImportError: - _real_asyncio = None +import asyncio as _real_asyncio try: import threading @@ -41,15 +38,16 @@ portage.proxy.lazyimport.lazyimport(globals(), ) from portage.util._eventloop.asyncio_event_loop import AsyncioEventLoop as _AsyncioEventLoop from portage.util._eventloop.global_event_loop import ( - _asyncio_enabled, global_event_loop as _global_event_loop, ) +# pylint: disable=redefined-builtin from portage.util.futures.futures import ( CancelledError, Future, InvalidStateError, TimeoutError, ) +# pylint: enable=redefined-builtin from portage.util.futures._asyncio.process import _Process from portage.util.futures._asyncio.tasks import ( ALL_COMPLETED, @@ -101,19 +99,14 @@ def get_event_loop(): def get_child_watcher(): - """Equivalent to calling get_event_loop_policy().get_child_watcher().""" - return get_event_loop_policy().get_child_watcher() + """Equivalent to calling get_event_loop_policy().get_child_watcher().""" + return get_event_loop_policy().get_child_watcher() def set_child_watcher(watcher): - """Equivalent to calling - get_event_loop_policy().set_child_watcher(watcher).""" - return get_event_loop_policy().set_child_watcher(watcher) - - -# Python 3.4 and later implement PEP 446, which makes newly -# created file descriptors non-inheritable by default. -_close_fds_default = sys.version_info < (3, 4) + """Equivalent to calling + get_event_loop_policy().set_child_watcher(watcher).""" + return get_event_loop_policy().set_child_watcher(watcher) def create_subprocess_exec(*args, **kwargs): @@ -138,8 +131,10 @@ def create_subprocess_exec(*args, **kwargs): @return: subset of asyncio.subprocess.Process interface """ loop = _wrap_loop(kwargs.pop('loop', None)) - kwargs.setdefault('close_fds', _close_fds_default) - if _asyncio_enabled and isinstance(loop._asyncio_wrapper, _AsyncioEventLoop): + # Python 3.4 and later implement PEP 446, which makes newly + # created file descriptors non-inheritable by default. + kwargs.setdefault('close_fds', False) + if isinstance(loop._asyncio_wrapper, _AsyncioEventLoop): # Use the real asyncio create_subprocess_exec (loop argument # is deprecated since since Python 3.8). return _real_asyncio.create_subprocess_exec(*args, **kwargs) @@ -163,7 +158,7 @@ def iscoroutinefunction(func): """ if _compat_coroutine._iscoroutinefunction(func): return True - elif _real_asyncio is not None and _real_asyncio.iscoroutinefunction(func): + if _real_asyncio.iscoroutinefunction(func): return True return False @@ -191,7 +186,7 @@ def ensure_future(coro_or_future, loop=None): @return: an instance of Future """ loop = _wrap_loop(loop) - if _asyncio_enabled and isinstance(loop._asyncio_wrapper, _AsyncioEventLoop): + if isinstance(loop._asyncio_wrapper, _AsyncioEventLoop): # Use the real asyncio loop and ensure_future. return _real_asyncio.ensure_future( coro_or_future, loop=loop._asyncio_wrapper._loop) @@ -240,18 +235,12 @@ def _wrap_loop(loop=None): @rtype: asyncio.AbstractEventLoop (or compatible) @return: event loop """ - return loop or _global_event_loop() - - -if _asyncio_enabled: # The default loop returned by _wrap_loop should be consistent # with global_event_loop, in order to avoid accidental registration # of callbacks with a loop that is not intended to run. - - def _wrap_loop(loop=None): - loop = loop or _global_event_loop() - return (loop if hasattr(loop, '_asyncio_wrapper') - else _AsyncioEventLoop(loop=loop)) + loop = loop or _global_event_loop() + return (loop if hasattr(loop, '_asyncio_wrapper') + else _AsyncioEventLoop(loop=loop)) def _safe_loop(): @@ -267,5 +256,4 @@ def _safe_loop(): """ if portage._internal_caller: return _global_event_loop() - else: - return _EventLoop(main=False) + return _EventLoop(main=False) diff --git a/lib/portage/util/futures/_asyncio/process.py b/lib/portage/util/futures/_asyncio/process.py index 020164c9b..6ff156c9d 100644 --- a/lib/portage/util/futures/_asyncio/process.py +++ b/lib/portage/util/futures/_asyncio/process.py @@ -1,15 +1,18 @@ -# Copyright 2018 Gentoo Foundation +# Copyright 2018-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 +import os + import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.util.futures:asyncio', + 'portage.util.futures.unix_events:_set_nonblocking', ) from portage.util.futures._asyncio.streams import _reader, _writer from portage.util.futures.compat_coroutine import coroutine, coroutine_return -class _Process(object): +class _Process: """ Emulate a subset of the asyncio.subprocess.Process interface, for python2. @@ -36,7 +39,7 @@ class _Process(object): return self._proc.returncode @coroutine - def communicate(self, input=None): + def communicate(self, input=None): # pylint: disable=redefined-builtin """ Read data from stdout and stderr, until end-of-file is reached. Wait for process to terminate. @@ -59,7 +62,11 @@ class _Process(object): if input is not None: if self._proc.stdin is None: raise TypeError('communicate: expected file or int, got {}'.format(type(self._proc.stdin))) - writer = asyncio.ensure_future(_writer(self._proc.stdin, input), loop=self._loop) + stdin = self._proc.stdin + stdin = os.fdopen(stdin, 'wb', 0) if isinstance(stdin, int) else stdin + _set_nonblocking(stdin.fileno()) + writer = asyncio.ensure_future(_writer(stdin, input, loop=self._loop), loop=self._loop) + writer.add_done_callback(lambda writer: stdin.close()) try: yield asyncio.wait(futures + [self.wait()], loop=self._loop) diff --git a/lib/portage/util/futures/_asyncio/streams.py b/lib/portage/util/futures/_asyncio/streams.py index 650a16491..ea5882dd3 100644 --- a/lib/portage/util/futures/_asyncio/streams.py +++ b/lib/portage/util/futures/_asyncio/streams.py @@ -1,4 +1,4 @@ -# Copyright 2018 Gentoo Foundation +# Copyright 2018-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import errno @@ -8,7 +8,6 @@ import portage portage.proxy.lazyimport.lazyimport(globals(), '_emerge.PipeReader:PipeReader', 'portage.util.futures:asyncio', - 'portage.util.futures.unix_events:_set_nonblocking', ) from portage.util.futures.compat_coroutine import coroutine @@ -31,7 +30,7 @@ def _reader(input_file, loop=None): return future -class _Reader(object): +class _Reader: def __init__(self, future, input_file, loop): self._future = future self._pipe_reader = PipeReader( @@ -59,38 +58,37 @@ class _Reader(object): @coroutine def _writer(output_file, content, loop=None): """ - Asynchronously write bytes to output file, and close it when - done. If an EnvironmentError other than EAGAIN is encountered, - which typically indicates that the other end of the pipe has - close, the error is raised. This function is a coroutine. + Asynchronously write bytes to output file. The output file is + assumed to be in non-blocking mode. If an EnvironmentError + other than EAGAIN is encountered, which typically indicates that + the other end of the pipe has closed, the error is raised. + This function is a coroutine. - @param output_file: output file descriptor - @type output_file: file or int + @param output_file: output file + @type output_file: file object @param content: content to write @type content: bytes @param loop: asyncio.AbstractEventLoop (or compatible) @type loop: event loop """ - fd = output_file if isinstance(output_file, int) else output_file.fileno() - _set_nonblocking(fd) loop = asyncio._wrap_loop(loop) - try: - while content: + fd = output_file.fileno() + while content: + try: + content = content[os.write(fd, content):] + except EnvironmentError as e: + if e.errno != errno.EAGAIN: + raise waiter = loop.create_future() - loop.add_writer(fd, lambda: waiter.set_result(None)) + loop.add_writer(fd, lambda: waiter.done() or waiter.set_result(None)) try: yield waiter - while content: - try: - content = content[os.write(fd, content):] - except EnvironmentError as e: - if e.errno == errno.EAGAIN: - break - else: - raise finally: - loop.remove_writer(fd) - except GeneratorExit: - raise - finally: - os.close(output_file) if isinstance(output_file, int) else output_file.close() + # The loop and output file may have been closed. + if not loop.is_closed(): + waiter.done() or waiter.cancel() + # Do not call remove_writer in cases where fd has + # been closed and then re-allocated to a concurrent + # coroutine as in bug 716636. + if not output_file.closed: + loop.remove_writer(fd) diff --git a/lib/portage/util/futures/_asyncio/tasks.py b/lib/portage/util/futures/_asyncio/tasks.py index b20765b7a..84c6f4462 100644 --- a/lib/portage/util/futures/_asyncio/tasks.py +++ b/lib/portage/util/futures/_asyncio/tasks.py @@ -8,12 +8,7 @@ ___all___ = ( 'wait', ) -try: - from asyncio import ALL_COMPLETED, FIRST_COMPLETED, FIRST_EXCEPTION -except ImportError: - ALL_COMPLETED = 'ALL_COMPLETED' - FIRST_COMPLETED ='FIRST_COMPLETED' - FIRST_EXCEPTION = 'FIRST_EXCEPTION' +from asyncio import ALL_COMPLETED, FIRST_COMPLETED, FIRST_EXCEPTION import portage portage.proxy.lazyimport.lazyimport(globals(), @@ -49,7 +44,7 @@ def wait(futures, loop=None, timeout=None, return_when=ALL_COMPLETED): return result_future -class _Waiter(object): +class _Waiter: def __init__(self, futures, timeout, return_when, result_future, loop): self._futures = futures self._completed = set() diff --git a/lib/portage/util/futures/compat_coroutine.py b/lib/portage/util/futures/compat_coroutine.py index 54fc316fe..79bd0da68 100644 --- a/lib/portage/util/futures/compat_coroutine.py +++ b/lib/portage/util/futures/compat_coroutine.py @@ -78,7 +78,7 @@ class _CoroutineReturnValue(Exception): self.result = result -class _GeneratorTask(object): +class _GeneratorTask: """ Asynchronously executes the generator to completion, waiting for the result of each Future that it yields, and sending the result @@ -134,4 +134,3 @@ class _GeneratorTask(object): else: self._current_task = asyncio.ensure_future(future, loop=self._loop) self._current_task.add_done_callback(self._next) - diff --git a/lib/portage/util/futures/events.py b/lib/portage/util/futures/events.py index b772bc242..85032fcdf 100644 --- a/lib/portage/util/futures/events.py +++ b/lib/portage/util/futures/events.py @@ -9,33 +9,29 @@ __all__ = ( import socket import subprocess -try: - from asyncio.events import ( - AbstractEventLoop as _AbstractEventLoop, - AbstractEventLoopPolicy as _AbstractEventLoopPolicy, - ) -except ImportError: - _AbstractEventLoop = object - _AbstractEventLoopPolicy = object +from asyncio.events import ( + AbstractEventLoop as _AbstractEventLoop, + AbstractEventLoopPolicy as _AbstractEventLoopPolicy, +) class AbstractEventLoopPolicy(_AbstractEventLoopPolicy): - """Abstract policy for accessing the event loop.""" + """Abstract policy for accessing the event loop.""" - def get_event_loop(self): - raise NotImplementedError + def get_event_loop(self): + raise NotImplementedError - def set_event_loop(self, loop): - raise NotImplementedError + def set_event_loop(self, loop): + raise NotImplementedError - def new_event_loop(self): - raise NotImplementedError + def new_event_loop(self): + raise NotImplementedError - def get_child_watcher(self): - raise NotImplementedError + def get_child_watcher(self): + raise NotImplementedError - def set_child_watcher(self, watcher): - raise NotImplementedError + def set_child_watcher(self, watcher): + raise NotImplementedError class AbstractEventLoop(_AbstractEventLoop): @@ -92,7 +88,7 @@ class AbstractEventLoop(_AbstractEventLoop): def set_default_executor(self, executor): raise NotImplementedError - def getaddrinfo(self, host, port, family=0, type=0, proto=0, flags=0): + def getaddrinfo(self, host, port, family=0, type=0, proto=0, flags=0): # pylint: disable=redefined-builtin raise NotImplementedError def getnameinfo(self, sockaddr, flags=0): @@ -188,4 +184,3 @@ class AbstractEventLoop(_AbstractEventLoop): def set_debug(self, enabled): raise NotImplementedError - diff --git a/lib/portage/util/futures/executor/fork.py b/lib/portage/util/futures/executor/fork.py index add7b3c9e..2cf713e9b 100644 --- a/lib/portage/util/futures/executor/fork.py +++ b/lib/portage/util/futures/executor/fork.py @@ -16,7 +16,7 @@ from portage.util.futures import asyncio from portage.util.cpuinfo import get_cpu_count -class ForkExecutor(object): +class ForkExecutor: """ An implementation of concurrent.futures.Executor that forks a new process for each task, with support for cancellation of tasks. diff --git a/lib/portage/util/futures/extendedfutures.py b/lib/portage/util/futures/extendedfutures.py index af384c745..158048a34 100644 --- a/lib/portage/util/futures/extendedfutures.py +++ b/lib/portage/util/futures/extendedfutures.py @@ -4,8 +4,6 @@ # This module provides an extended subset of the asyncio.futures.Futures # interface. -from __future__ import unicode_literals - __all__ = ( 'CancelledError', 'ExtendedFuture', diff --git a/lib/portage/util/futures/futures.py b/lib/portage/util/futures/futures.py index 9c9900d4c..839c767a7 100644 --- a/lib/portage/util/futures/futures.py +++ b/lib/portage/util/futures/futures.py @@ -5,8 +5,6 @@ # asyncio module (Python 3.3 and earlier), this module provides a # subset of the asyncio.futures.Futures interface. -from __future__ import unicode_literals - __all__ = ( 'CancelledError', 'Future', @@ -14,32 +12,14 @@ __all__ = ( 'TimeoutError', ) -try: - from asyncio import ( - CancelledError, - Future, - InvalidStateError, - TimeoutError, - ) -except ImportError: - - from portage.exception import PortageException - - class Error(PortageException): - pass - - class CancelledError(Error): - def __init__(self): - Error.__init__(self, "cancelled") - - class TimeoutError(Error): - def __init__(self): - Error.__init__(self, "timed out") - - class InvalidStateError(Error): - pass - - Future = None +# pylint: disable=redefined-builtin +from asyncio import ( + CancelledError, + Future, + InvalidStateError, + TimeoutError, +) +# pylint: enable=redefined-builtin import portage portage.proxy.lazyimport.lazyimport(globals(), @@ -50,7 +30,7 @@ _PENDING = 'PENDING' _CANCELLED = 'CANCELLED' _FINISHED = 'FINISHED' -class _EventLoopFuture(object): +class _EventLoopFuture: """ This class provides (a subset of) the asyncio.Future interface, for use with the EventLoop class, because EventLoop is currently @@ -193,7 +173,3 @@ class _EventLoopFuture(object): self._exception = exception self._state = _FINISHED self._schedule_callbacks() - - -if Future is None: - Future = _EventLoopFuture diff --git a/lib/portage/util/futures/retry.py b/lib/portage/util/futures/retry.py index ccfc087ab..4092f60d6 100644 --- a/lib/portage/util/futures/retry.py +++ b/lib/portage/util/futures/retry.py @@ -73,7 +73,7 @@ def _retry(loop, try_max, try_timeout, overall_timeout, delay_func, return future -class _Retry(object): +class _Retry: def __init__(self, future, loop, try_max, try_timeout, overall_timeout, delay_func, reraise, func): self._future = future diff --git a/lib/portage/util/futures/transports.py b/lib/portage/util/futures/transports.py index 60ea93073..016ecbef8 100644 --- a/lib/portage/util/futures/transports.py +++ b/lib/portage/util/futures/transports.py @@ -1,10 +1,7 @@ # Copyright 2018 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -try: - from asyncio.transports import Transport as _Transport -except ImportError: - _Transport = object +from asyncio.transports import Transport as _Transport class _FlowControlMixin(_Transport): diff --git a/lib/portage/util/futures/unix_events.py b/lib/portage/util/futures/unix_events.py index 3381eaa7d..16a9e12b7 100644 --- a/lib/portage/util/futures/unix_events.py +++ b/lib/portage/util/futures/unix_events.py @@ -6,20 +6,13 @@ __all__ = ( 'DefaultEventLoopPolicy', ) -try: - import asyncio as _real_asyncio - from asyncio.base_subprocess import BaseSubprocessTransport as _BaseSubprocessTransport - from asyncio.unix_events import AbstractChildWatcher as _AbstractChildWatcher - from asyncio.transports import ( - ReadTransport as _ReadTransport, - WriteTransport as _WriteTransport, - ) -except ImportError: - _real_asyncio = None - _AbstractChildWatcher = object - _BaseSubprocessTransport = object - _ReadTransport = object - _WriteTransport = object +import asyncio as _real_asyncio +from asyncio.base_subprocess import BaseSubprocessTransport as _BaseSubprocessTransport +from asyncio.unix_events import AbstractChildWatcher as _AbstractChildWatcher +from asyncio.transports import ( + ReadTransport as _ReadTransport, + WriteTransport as _WriteTransport, +) import errno import fcntl @@ -32,7 +25,6 @@ import subprocess import sys from portage.util._eventloop.global_event_loop import ( - _asyncio_enabled, global_event_loop as _global_event_loop, ) from portage.util.futures import ( @@ -441,7 +433,7 @@ class _UnixWritePipeTransport(_FlowControlMixin, _WriteTransport): return if n == len(data): return - elif n > 0: + if n > 0: data = memoryview(data)[n:] self._loop.add_writer(self._fileno, self._write_ready) @@ -471,7 +463,7 @@ class _UnixWritePipeTransport(_FlowControlMixin, _WriteTransport): self._loop.remove_reader(self._fileno) self._call_connection_lost(None) return - elif n > 0: + if n > 0: del self._buffer[:n] def can_write_eof(self): @@ -625,10 +617,9 @@ class _PortageChildWatcher(_AbstractChildWatcher): def _compute_returncode(self, status): if os.WIFSIGNALED(status): return -os.WTERMSIG(status) - elif os.WIFEXITED(status): + if os.WIFEXITED(status): return os.WEXITSTATUS(status) - else: - return status + return status def add_child_handler(self, pid, callback, *args): """ @@ -701,5 +692,4 @@ class _AsyncioEventLoopPolicy(_PortageEventLoopPolicy): return super(_AsyncioEventLoopPolicy, self).get_child_watcher() -DefaultEventLoopPolicy = (_AsyncioEventLoopPolicy if _asyncio_enabled - else _PortageEventLoopPolicy) +DefaultEventLoopPolicy = _AsyncioEventLoopPolicy diff --git a/lib/portage/util/install_mask.py b/lib/portage/util/install_mask.py index 0013effa1..9442128bd 100644 --- a/lib/portage/util/install_mask.py +++ b/lib/portage/util/install_mask.py @@ -1,4 +1,4 @@ -# Copyright 2018-2019 Gentoo Authors +# Copyright 2018-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 __all__ = ['install_mask_dir', 'InstallMask'] @@ -6,9 +6,7 @@ __all__ = ['install_mask_dir', 'InstallMask'] import collections import errno import fnmatch -import functools import operator -import sys from portage import os, _unicode_decode from portage.exception import ( @@ -20,11 +18,6 @@ from portage.exception import ( ) from portage.util import normalize_path -if sys.hexversion >= 0x3000000: - _unicode = str -else: - _unicode = unicode - def _defaultdict_tree(): return collections.defaultdict(_defaultdict_tree) @@ -38,7 +31,7 @@ _pattern = collections.namedtuple('_pattern', ( )) -class InstallMask(object): +class InstallMask: def __init__(self, install_mask): """ @param install_mask: INSTALL_MASK value @@ -152,7 +145,7 @@ def _raise_exc(e): wrapper_cls = _exc_map.get(e.errno) if wrapper_cls is None: raise - wrapper = wrapper_cls(_unicode(e)) + wrapper = wrapper_cls(str(e)) wrapper.__cause__ = e raise wrapper diff --git a/lib/portage/util/iterators/MultiIterGroupBy.py b/lib/portage/util/iterators/MultiIterGroupBy.py index 2c31f269f..18f48a016 100644 --- a/lib/portage/util/iterators/MultiIterGroupBy.py +++ b/lib/portage/util/iterators/MultiIterGroupBy.py @@ -3,7 +3,7 @@ import bisect -class MultiIterGroupBy(object): +class MultiIterGroupBy: """ This class functions similarly to the itertools.groupby function, except that it takes multiple source iterators as input. The source @@ -78,7 +78,7 @@ class MultiIterGroupBy(object): for k in yield_these: yield key_map.pop(k) -class _IteratorTracker(object): +class _IteratorTracker: __slots__ = ('current', 'iterator') diff --git a/lib/portage/util/lafilefixer.py b/lib/portage/util/lafilefixer.py index 110010363..482762bf7 100644 --- a/lib/portage/util/lafilefixer.py +++ b/lib/portage/util/lafilefixer.py @@ -109,18 +109,18 @@ def rewrite_lafile(contents): #Two cases: #1) /usr/lib64/libfoo.la, turn it into -lfoo and append -L/usr/lib64 to libladir #2) libfoo.la, keep it - dir, file = _os.path.split(dep_libs_entry) + dirname, basename = _os.path.split(dep_libs_entry) - if not dir or not file.startswith(b"lib"): + if not dirname or not basename.startswith(b"lib"): if dep_libs_entry not in new_dep_libs: new_dep_libs.append(dep_libs_entry) else: #/usr/lib64/libfoo.la -> -lfoo - lib = b"-l" + file[3:-3] + lib = b"-l" + basename[3:-3] if lib not in new_dep_libs: new_dep_libs.append(lib) #/usr/lib64/libfoo.la -> -L/usr/lib64 - ladir = b"-L" + dir + ladir = b"-L" + dirname if ladir not in libladir: libladir.append(ladir) @@ -181,5 +181,4 @@ def rewrite_lafile(contents): if changed: return True, contents - else: - return False, None + return False, None diff --git a/lib/portage/util/listdir.py b/lib/portage/util/listdir.py index 2012e145f..e6fb9c06d 100644 --- a/lib/portage/util/listdir.py +++ b/lib/portage/util/listdir.py @@ -5,10 +5,7 @@ __all__ = ['cacheddir', 'listdir'] import errno import stat -import sys -if sys.hexversion < 0x3000000: - from itertools import izip as zip from portage import os from portage.const import VCS_DIRS diff --git a/lib/portage/util/locale.py b/lib/portage/util/locale.py index 5b09945d6..f7e99a4fd 100644 --- a/lib/portage/util/locale.py +++ b/lib/portage/util/locale.py @@ -6,7 +6,7 @@ Function to check whether the current used LC_CTYPE handles case transformations of ASCII characters in a way compatible with the POSIX locale. """ -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import import locale import logging diff --git a/lib/portage/util/monotonic.py b/lib/portage/util/monotonic.py deleted file mode 100644 index e50564851..000000000 --- a/lib/portage/util/monotonic.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2018 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -__all__ = ['monotonic'] - -import time -try: - import threading -except ImportError: - import dummy_threading as threading - -monotonic = getattr(time, 'monotonic', None) - -if monotonic is None: - def monotonic(): - """ - Emulate time.monotonic() which is available in Python 3.3 and later. - - @return: A float expressed in seconds since an epoch. - """ - with monotonic._lock: - current = time.time() + monotonic._offset - delta = current - monotonic._previous - if delta < 0: - monotonic._offset -= delta - current = monotonic._previous - else: - monotonic._previous = current - return current - - # offset is used to counteract any backward movements - monotonic._offset = 0 - monotonic._previous = time.time() - monotonic._lock = threading.Lock() diff --git a/lib/portage/util/movefile.py b/lib/portage/util/movefile.py index 5477a669f..3a17d5240 100644 --- a/lib/portage/util/movefile.py +++ b/lib/portage/util/movefile.py @@ -1,7 +1,7 @@ -# Copyright 2010-2018 Gentoo Foundation +# Copyright 2010-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import __all__ = ['movefile'] @@ -9,7 +9,6 @@ import errno import fnmatch import os as _os import stat -import sys import textwrap import portage @@ -41,7 +40,7 @@ def _get_xattr_excluder(pattern): return value -class _xattr_excluder(object): +class _xattr_excluder: __slots__ = ('_pattern_split',) @@ -188,18 +187,13 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, except OSError: pass - if sys.hexversion >= 0x3030000: - try: - os.utime(dest, ns=(sstat.st_mtime_ns, sstat.st_mtime_ns), follow_symlinks=False) - except NotImplementedError: - # utimensat() and lutimes() missing in libc. - return os.stat(dest, follow_symlinks=False).st_mtime_ns - else: - return sstat.st_mtime_ns + try: + os.utime(dest, ns=(sstat.st_mtime_ns, sstat.st_mtime_ns), follow_symlinks=False) + except NotImplementedError: + # utimensat() and lutimes() missing in libc. + return os.stat(dest, follow_symlinks=False).st_mtime_ns else: - # utime() in Python <3.3 only works on the target of a symlink, so it's not - # possible to preserve mtime on symlinks. - return os.lstat(dest)[stat.ST_MTIME] + return sstat.st_mtime_ns except SystemExit as e: raise except Exception as e: @@ -312,49 +306,26 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, # if the nanosecond part of the timestamp is 999999881 ns or greater. try: if hardlinked: - if sys.hexversion >= 0x3030000: - newmtime = os.stat(dest).st_mtime_ns - else: - newmtime = os.stat(dest)[stat.ST_MTIME] + newmtime = os.stat(dest).st_mtime_ns else: # Note: It is not possible to preserve nanosecond precision # (supported in POSIX.1-2008 via utimensat) with the IEEE 754 # double precision float which only has a 53 bit significand. if newmtime is not None: - if sys.hexversion >= 0x3030000: - os.utime(dest, ns=(newmtime, newmtime)) - else: - os.utime(dest, (newmtime, newmtime)) + os.utime(dest, ns=(newmtime, newmtime)) else: - if sys.hexversion >= 0x3030000: - newmtime = sstat.st_mtime_ns - else: - newmtime = sstat[stat.ST_MTIME] + newmtime = sstat.st_mtime_ns if renamefailed: - if sys.hexversion >= 0x3030000: - # If rename succeeded then timestamps are automatically - # preserved with complete precision because the source - # and destination inodes are the same. Otherwise, manually - # update timestamps with nanosecond precision. - os.utime(dest, ns=(newmtime, newmtime)) - else: - # If rename succeeded then timestamps are automatically - # preserved with complete precision because the source - # and destination inodes are the same. Otherwise, round - # down to the nearest whole second since python's float - # st_mtime cannot be used to preserve the st_mtim.tv_nsec - # field with complete precision. Note that we have to use - # stat_obj[stat.ST_MTIME] here because the float - # stat_obj.st_mtime rounds *up* sometimes. - os.utime(dest, (newmtime, newmtime)) + # If rename succeeded then timestamps are automatically + # preserved with complete precision because the source + # and destination inodes are the same. Otherwise, manually + # update timestamps with nanosecond precision. + os.utime(dest, ns=(newmtime, newmtime)) except OSError: # The utime can fail here with EPERM even though the move succeeded. # Instead of failing, use stat to return the mtime if possible. try: - if sys.hexversion >= 0x3030000: - newmtime = os.stat(dest).st_mtime_ns - else: - newmtime = os.stat(dest)[stat.ST_MTIME] + newmtime = os.stat(dest).st_mtime_ns except OSError as e: writemsg(_("!!! Failed to stat in movefile()\n"), noiselevel=-1) writemsg("!!! %s\n" % dest, noiselevel=-1) diff --git a/lib/portage/util/mtimedb.py b/lib/portage/util/mtimedb.py index 30922a901..b8ceeff8e 100644 --- a/lib/portage/util/mtimedb.py +++ b/lib/portage/util/mtimedb.py @@ -1,4 +1,4 @@ -# Copyright 2010-2012 Gentoo Foundation +# Copyright 2010-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 __all__ = ['MtimeDB'] @@ -12,7 +12,6 @@ except ImportError: import errno import io import json -import sys import portage from portage import _encodings @@ -32,9 +31,6 @@ class MtimeDB(dict): "indent": "\t", "sort_keys": True } - if sys.hexversion < 0x30200F0: - # indent only supports int number of spaces - _json_write_opts["indent"] = 4 def __init__(self, filename): dict.__init__(self) diff --git a/lib/portage/util/socks5.py b/lib/portage/util/socks5.py index 86bb24f25..c0657ae2a 100644 --- a/lib/portage/util/socks5.py +++ b/lib/portage/util/socks5.py @@ -15,7 +15,7 @@ from portage.util.futures.compat_coroutine import coroutine from portage.util.futures import asyncio -class ProxyManager(object): +class ProxyManager: """ A class to start and control a single running SOCKSv5 server process for Portage. diff --git a/lib/portage/util/whirlpool.py b/lib/portage/util/whirlpool.py index 170ae73f8..1071d5155 100644 --- a/lib/portage/util/whirlpool.py +++ b/lib/portage/util/whirlpool.py @@ -25,9 +25,7 @@ ## ## This Python implementation is therefore also placed in the public domain. -import sys -if sys.hexversion >= 0x3000000: - xrange = range +# pylint: disable=mixed-indentation #block_size = 64 digest_size = 64 @@ -641,8 +639,6 @@ def WhirlpoolInit(ctx): def WhirlpoolAdd(source, sourceBits, ctx): if not isinstance(source, bytes): raise TypeError("Expected %s, got %s" % (bytes, type(source))) - if sys.hexversion < 0x3000000: - source = [ord(s)&0xff for s in source] carry = 0 value = sourceBits @@ -700,19 +696,19 @@ def WhirlpoolFinalize(ctx): bufferPos += 1 if bufferPos > 32: if bufferPos < 64: - for i in xrange(64 - bufferPos): + for i in range(64 - bufferPos): ctx.buffer[bufferPos+i] = 0 processBuffer(ctx) bufferPos = 0 if bufferPos < 32: - for i in xrange(32 - bufferPos): + for i in range(32 - bufferPos): ctx.buffer[bufferPos+i] = 0 bufferPos = 32 - for i in xrange(32): + for i in range(32): ctx.buffer[32+i] = ctx.bitLength[i] processBuffer(ctx) digest = '' - for i in xrange(8): + for i in range(8): digest += chr((ctx.hash[i] >> 56) % 0x100) digest += chr((ctx.hash[i] >> 48) % 0x100) digest += chr((ctx.hash[i] >> 40) % 0x100) @@ -743,7 +739,7 @@ def processBuffer(ctx): buffr = ctx.buffer buf_cnt = 0 - for i in xrange(8): + for i in range(8): block[i] = ((buffr[buf_cnt+0] & 0xff) << 56) ^ \ ((buffr[buf_cnt+1] & 0xff) << 48) ^ \ ((buffr[buf_cnt+2] & 0xff) << 40) ^ \ @@ -753,11 +749,11 @@ def processBuffer(ctx): ((buffr[buf_cnt+6] & 0xff) << 8) ^ \ ((buffr[buf_cnt+7] & 0xff) << 0) buf_cnt += 8 - for i in xrange(8): + for i in range(8): K[i] = ctx.hash[i] state[i] = block[i] ^ K[i] - for r in xrange(1, R+1): + for r in range(1, R+1): L[0] = CDo(K, 0, 7, 6, 5, 4, 3, 2, 1) ^ rc[r] L[1] = CDo(K, 1, 0, 7, 6, 5, 4, 3, 2) L[2] = CDo(K, 2, 1, 0, 7, 6, 5, 4, 3) @@ -766,7 +762,7 @@ def processBuffer(ctx): L[5] = CDo(K, 5, 4, 3, 2, 1, 0, 7, 6) L[6] = CDo(K, 6, 5, 4, 3, 2, 1, 0, 7) L[7] = CDo(K, 7, 6, 5, 4, 3, 2, 1, 0) - for i in xrange(8): + for i in range(8): K[i] = L[i] L[0] = CDo(state, 0, 7, 6, 5, 4, 3, 2, 1) ^ K[0] L[1] = CDo(state, 1, 0, 7, 6, 5, 4, 3, 2) ^ K[1] @@ -776,10 +772,10 @@ def processBuffer(ctx): L[5] = CDo(state, 5, 4, 3, 2, 1, 0, 7, 6) ^ K[5] L[6] = CDo(state, 6, 5, 4, 3, 2, 1, 0, 7) ^ K[6] L[7] = CDo(state, 7, 6, 5, 4, 3, 2, 1, 0) ^ K[7] - for i in xrange(8): + for i in range(8): state[i] = L[i] # apply the Miyaguchi-Preneel compression function - for i in xrange(8): + for i in range(8): ctx.hash[i] ^= state[i] ^ block[i] return diff --git a/lib/portage/util/writeable_check.py b/lib/portage/util/writeable_check.py index e5b14c023..e3c946e8e 100644 --- a/lib/portage/util/writeable_check.py +++ b/lib/portage/util/writeable_check.py @@ -9,8 +9,6 @@ accepts a list of directories and returns a list of mounts which need to be remounted RW, then add "elif ostype == (the ostype value for your OS)" to get_ro_checker(). """ -from __future__ import unicode_literals - import io import logging import os diff --git a/lib/portage/versions.py b/lib/portage/versions.py index 0b1d50e7c..d4ab9d199 100644 --- a/lib/portage/versions.py +++ b/lib/portage/versions.py @@ -2,8 +2,6 @@ # Copyright 1998-2016 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals - __all__ = [ 'best', 'catpkgsplit', 'catsplit', 'cpv_getkey', 'cpv_getversion', 'cpv_sort_key', 'pkgcmp', 'pkgsplit', @@ -11,14 +9,9 @@ __all__ = [ ] import re -import sys import warnings +from functools import lru_cache -if sys.hexversion < 0x3000000: - _unicode = unicode -else: - _unicode = str - long = int import portage portage.proxy.lazyimport.lazyimport(globals(), @@ -113,11 +106,11 @@ def _get_pv_re(eapi_attrs): def ververify(myver, silent=1): if ver_regexp.match(myver): return True - else: - if not silent: - print(_("!!! syntax error in version: %s") % myver) - return False + if not silent: + print(_("!!! syntax error in version: %s") % myver) + return False +@lru_cache(1024) def vercmp(ver1, ver2, silent=1): """ Compare two versions @@ -210,9 +203,9 @@ def vercmp(ver1, ver2, silent=1): for i in range(0, max(len(list1), len(list2))): if len(list1) <= i: return -1 - elif len(list2) <= i: + if len(list2) <= i: return 1 - elif list1[i] != list2[i]: + if list1[i] != list2[i]: a = list1[i] b = list2[i] rval = (a > b) - (a < b) @@ -347,6 +340,7 @@ def _pkgsplit(mypkg, eapi=None): _cat_re = re.compile('^%s$' % _cat, re.UNICODE) _missing_cat = 'null' +@lru_cache(10240) def catpkgsplit(mydata, silent=1, eapi=None): """ Takes a Category/Package-Version-Rev and returns a list of each. @@ -379,7 +373,7 @@ def catpkgsplit(mydata, silent=1, eapi=None): retval = (cat, p_split[0], p_split[1], p_split[2]) return retval -class _pkg_str(_unicode): +class _pkg_str(str): """ This class represents a cpv. It inherits from str (unicode in python2) and has attributes that cache results for use by functions like catpkgsplit and @@ -398,15 +392,15 @@ class _pkg_str(_unicode): def __new__(cls, cpv, metadata=None, settings=None, eapi=None, repo=None, slot=None, build_time=None, build_id=None, file_size=None, mtime=None, db=None): - return _unicode.__new__(cls, cpv) + return str.__new__(cls, cpv) def __init__(self, cpv, metadata=None, settings=None, eapi=None, repo=None, slot=None, build_time=None, build_id=None, file_size=None, mtime=None, db=None): - if not isinstance(cpv, _unicode): - # Avoid TypeError from _unicode.__init__ with PyPy. + if not isinstance(cpv, str): + # Avoid TypeError from str.__init__ with PyPy. cpv = _unicode_decode(cpv) - _unicode.__init__(cpv) + str.__init__(cpv) if metadata is not None: self.__dict__['_metadata'] = metadata slot = metadata.get('SLOT', slot) @@ -471,7 +465,7 @@ class _pkg_str(_unicode): def _long(var, default): if var is not None: try: - var = long(var) + var = int(var) except ValueError: if var: var = -1 @@ -511,8 +505,7 @@ def pkgsplit(mypkg, silent=1, eapi=None): cat, pn, ver, rev = catpsplit if cat is _missing_cat and '/' not in mypkg: return (pn, ver, rev) - else: - return (cat + '/' + pn, ver, rev) + return (cat + '/' + pn, ver, rev) def cpv_getkey(mycpv, eapi=None): """Calls catpkgsplit on a cpv and returns only the cp.""" @@ -535,8 +528,7 @@ def cpv_getkey(mycpv, eapi=None): mylen = len(myslash) if mylen == 2: return myslash[0] + "/" + mysplit[0] - else: - return mysplit[0] + return mysplit[0] def cpv_getversion(mycpv, eapi=None): """Returns the v (including revision) from an cpv.""" diff --git a/lib/portage/xml/metadata.py b/lib/portage/xml/metadata.py index 64246c828..646edb5b9 100644 --- a/lib/portage/xml/metadata.py +++ b/lib/portage/xml/metadata.py @@ -1,4 +1,4 @@ -# Copyright 2010-2019 Gentoo Authors +# Copyright 2010-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 """Provides an easy-to-use python interface to Gentoo's metadata.xml file. @@ -28,11 +28,8 @@ 'Thomas Mills Hinkle' """ -from __future__ import unicode_literals - __all__ = ('MetaDataXML', 'parse_metadata_use') -import sys try: import xml.etree.cElementTree as etree @@ -55,10 +52,6 @@ import xml.etree.ElementTree from portage import _encodings, _unicode_encode from portage.util import cmp_sort_key, unique_everseen -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - basestring = str - class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder): """ @@ -68,7 +61,7 @@ class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder): def doctype(self, name, pubid, system): pass -class _Maintainer(object): +class _Maintainer: """An object for representing one maintainer. @type email: str or None @@ -101,7 +94,7 @@ class _Maintainer(object): return "<%s %r>" % (self.__class__.__name__, self.email) -class _Useflag(object): +class _Useflag: """An object for representing one USE flag. @todo: Is there any way to have a keyword option to leave in @@ -131,7 +124,7 @@ class _Useflag(object): return "<%s %r>" % (self.__class__.__name__, self.name) -class _Upstream(object): +class _Upstream: """An object for representing one package's upstream. @type maintainers: list @@ -187,7 +180,7 @@ class _Upstream(object): return [(e.text, e.get('type')) for e in self.node.findall('remote-id') if e.text] -class MetaDataXML(object): +class MetaDataXML: """Access metadata.xml""" def __init__(self, metadata_xml_path, herds): @@ -479,12 +472,12 @@ def parse_metadata_use(xml_tree): stack.append(flag) while stack: obj = stack.pop() - if isinstance(obj, basestring): + if isinstance(obj, str): inner_text.append(obj) continue - if isinstance(obj.text, basestring): + if isinstance(obj.text, str): inner_text.append(obj.text) - if isinstance(obj.tail, basestring): + if isinstance(obj.tail, str): stack.append(obj.tail) stack.extend(reversed(obj)) @@ -494,4 +487,3 @@ def parse_metadata_use(xml_tree): # (flag_restrict can be None) uselist[flag.get("name")][flag_restrict] = " ".join("".join(inner_text).split()) return uselist - diff --git a/lib/portage/xpak.py b/lib/portage/xpak.py index c708190b9..3401c5073 100644 --- a/lib/portage/xpak.py +++ b/lib/portage/xpak.py @@ -24,7 +24,6 @@ __all__ = [ import array import errno -import sys import portage from portage import os @@ -78,8 +77,6 @@ def encodeint(myint): def decodeint(mystring): """Takes a 4 byte string and converts it into a 4 byte integer. Returns an integer.""" - if sys.hexversion < 0x3000000: - mystring = [ord(x) for x in mystring] myint = 0 myint += mystring[3] myint += mystring[2] << 8 @@ -276,7 +273,7 @@ def xpand(myid, mydest): mydat.close() startpos = startpos + namelen + 12 -class tbz2(object): +class tbz2: def __init__(self, myfile): self.file = myfile self.filestat = None diff --git a/man/ebuild.5 b/man/ebuild.5 index 4ca3f3a15..476844281 100644 --- a/man/ebuild.5 +++ b/man/ebuild.5 @@ -342,7 +342,7 @@ This variable must NEVER be modified. \fBExample\fR: .nf - x11\-base/xorg\-server\-1.20.5\-r2 \-\-> '\fIx11\-base/xorg\-server\-1.20.5\fR' + x11\-base/xorg\-server\-1.20.5\-r2 \-\-> '\fIxorg\-server\-1.20.5\fR' .fi .TP .B PN @@ -350,7 +350,7 @@ Contains the name of the script without the version number. \fBExample\fR: .nf - x11\-base/xorg\-server\-1.20.5\-r2 \-\-> '\fIx11\-base/xorg\-server\fR' + x11\-base/xorg\-server\-1.20.5\-r2 \-\-> '\fIxorg\-server\fR' .fi .TP .B PV diff --git a/man/egencache.1 b/man/egencache.1 index 27cd69e63..98b230a14 100644 --- a/man/egencache.1 +++ b/man/egencache.1 @@ -13,7 +13,7 @@ itself, for distribution. .BR "\-\-update [ATOM] ... " Update the \fImetadata/md5\-cache/\fR directory (generate metadata as necessary). -If no package atoms are specified then all will be updated. See ebuild(5) +If no package atoms are specified then all will be updated. See \fBebuild\fR(5) for the details on package atom syntax. .TP .BR "\-\-update\-changelogs" @@ -151,7 +151,7 @@ exists in the repository. It can also be explicitly enabled via the cache\-formats setting in \fImetadata/layout.conf\fR (refer to \fBportage\fR(5) for example usage). If the 'pms' cache format is enabled and the 'md5-dict' format is not enabled, then it is necessary to enable -\fBmetadata-transfer\fR in \fBFEATURES\fR (see \fBmake.conf(5)\fR). +\fBmetadata-transfer\fR in \fBFEATURES\fR (see \fBmake.conf\fR(5)). This causes intermediate cache (in a different format that includes eclass state) to be generated inside the directory which is configurable via the \fB\-\-cache\-dir\fR option. diff --git a/man/emerge.1 b/man/emerge.1 index aa28ab337..a2a0c2faa 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -56,8 +56,8 @@ A \fItbz2file\fR must be a valid .tbz2 created with \fBebuild A \fIfile\fR must be a file or directory that has been installed by one or more packages. If an absolute path is not used, then it must begin with either "./" or "../". For directories that are owned by multiple packages, all -owning packages will be selected. See the portageq(1) owners command if you -would like to query the owners of one or more files or directories. +owning packages will be selected. See the \fBportageq\fR(1) owners command if +you would like to query the owners of one or more files or directories. .TP .BR set A \fIset\fR is a convenient shorthand for a large group of @@ -1100,7 +1100,7 @@ This also applies to options that enable the \fB\-\-usepkg\fR option implicitly, such as \fB\-\-getbinpkg\fR. This setting can be added to -\fBEMERGE_DEFAULT_OPTS\fR (see make.conf(5)) and later overridden via the +\fBEMERGE_DEFAULT_OPTS\fR (see \fBmake.conf\fR(5)) and later overridden via the command line. .TP .BR "\-\-with\-bdeps\-auto < y | n >" @@ -1109,7 +1109,7 @@ This option is used to enable or disable the program logic that causes actions. This option is enabled by default. Use \fB\-\-with\-bdeps\-auto=n\fR to prevent \fB\-\-with\-bdeps\fR from being automatically enabled for installation actions. This setting can -be added to \fBEMERGE_DEFAULT_OPTS\fR (see make.conf(5)) and later +be added to \fBEMERGE_DEFAULT_OPTS\fR (see \fBmake.conf\fR(5)) and later overridden via the command line. \fBNOTE:\fR The program logic that causes \fB\-\-with\-bdeps\fR to be diff --git a/man/make.conf.5 b/man/make.conf.5 index ab00cb7d7..d0af57891 100644 --- a/man/make.conf.5 +++ b/man/make.conf.5 @@ -1,4 +1,4 @@ -.TH "MAKE.CONF" "5" "May 2020" "Portage VERSION" "Portage" +.TH "MAKE.CONF" "5" "Jun 2020" "Portage VERSION" "Portage" .SH "NAME" make.conf \- custom settings for Portage .SH "SYNOPSIS" @@ -985,6 +985,11 @@ with an integer pid. For example, a value of "ionice \-c 3 \-p \\${PID}" will set idle io priority. For more information about ionice, see \fBionice\fR(1). This variable is unset by default. .TP +.B PORTAGE_LOG_FILTER_FILE_CMD +This variable specifies a command that filters build log output to a +log file. In order to filter ANSI escape codes from build logs, +\fBansifilter\fR(1) is a convenient setting for this variable. +.TP .B PORTAGE_LOGDIR This variable defines the directory in which per\-ebuild logs are kept. Logs are created only when this is set. They are stored as diff --git a/man/portage.5 b/man/portage.5 index 36c871123..a7e64cd5f 100644 --- a/man/portage.5 +++ b/man/portage.5 @@ -1,4 +1,4 @@ -.TH "PORTAGE" "5" "Apr 2019" "Portage VERSION" "Portage" +.TH "PORTAGE" "5" "Jul 2020" "Portage VERSION" "Portage" .SH NAME portage \- the heart of Gentoo .SH "DESCRIPTION" @@ -916,13 +916,11 @@ When 'force = aliases' attribute is not set, \fBegencache\fR(1), since operations performed by these tools are inherently \fBnot\fR \fIsite\-specific\fR. .TP -.B auto\-sync +.B auto\-sync = yes|no|true|false This setting determines if the repo will be synced during "\fBemerge \-\-sync\fR" or "\fBemaint sync \-\-auto\fR" runs. This allows for repositories to be synced only when desired via "\fBemaint sync \-\-repo foo\fR". .br -Valid values: yes, no, true, false. -.br If unset, the repo will be treated as set yes, true. .TP @@ -966,20 +964,18 @@ since operations performed by these tools are inherently .B priority Specifies priority of given repository. .TP -.B strict\-misc\-digests +.B strict\-misc\-digests = yes|no|true|false This setting determines whether digests are checked for files declared in the Manifest with MISC type (includes ChangeLog and metadata.xml -files). Defaults to true. -.br -Valid values: true, false. +files). Defaults to yes, true. .TP -.B sync\-allow\-hardlinks = yes|no +.B sync\-allow\-hardlinks = yes|no|true|false Allow sync plugins to use hardlinks in order to ensure that a repository remains in a valid state if something goes wrong during the sync operation. For example, if signature verification fails during a sync operation, the previous state of the repository will be preserved. This option may conflict with configurations that restrict the use of hardlinks, such as -overlay filesystems. +overlay filesystems. Defaults to yes, true. .TP .B sync\-cvs\-repo Specifies CVS repository. @@ -1016,16 +1012,17 @@ See also example for sync-git-clone-env. .B sync\-git\-pull\-extra\-opts Extra options to give to git when updating repository (git pull). .TP -.B sync\-git\-verify\-commit\-signature = true|false +.B sync\-git\-verify\-commit\-signature = yes|no|true|false Require the top commit in the repository to contain a good OpenPGP -signature. Defaults to false. +signature. Defaults to no, false. .TP -.B sync\-hooks\-only\-on\-change +.B sync\-hooks\-only\-on\-change = yes|no|true|false If set to true, then sync of a given repository will not trigger postsync hooks unless hooks would have executed for a master repository or the -repository has changed since the previous sync operation. +repository has changed since the previous sync operation. Defaults to +no, false. .TP -.B sync\-rcu = yes|no +.B sync\-rcu = yes|no|true|false Enable read\-copy\-update (RCU) behavior for sync operations. The current latest immutable version of a repository will be referenced by a symlink found where the repository would normally be located (see the \fBlocation\fR @@ -1125,6 +1122,13 @@ only for protocols supporting cryptographic verification, provided that the respective verification option is enabled. If unset, the user's keyring is used. .TP +.B sync\-openpgp\-key\-refresh = yes +Enable OpenPGP key(ring) refresh. This option is enabled by default. + +\fBWarning\fR: It is a security vulnerability to disable this option +because this will prevent detection of revoked keys! + +.TP .B sync\-openpgp\-key\-refresh\-retry\-count = 40 Maximum number of times to retry key refresh if it fails. Between each key refresh attempt, there is an exponential delay with a constant @@ -1149,10 +1153,10 @@ Pass \fIname\fR as the `gpg \-\-keyserver` argument. Refer to the \fBgpg\fR(1) man page for information about the `gpg \-\-keyserver` \fIname\fR format. .TP -.B sync-rsync-vcs-ignore = true|false +.B sync\-rsync\-vcs\-ignore = yes|no|true|false Ignore vcs directories that may be present in the repository. It is the user's responsibility to set sync-rsync-extra-opts to protect vcs -directories if appropriate. +directories if appropriate. Defaults to no, false. .TP .B sync\-rsync\-verify\-jobs = 1 Number of parallel jobs to use when verifying nested Manifests. When @@ -1164,21 +1168,21 @@ Defaults to 1. Warn if repository is older than the specified number of days. Disabled when 0. Defaults to disabled. .TP -.B sync\-rsync\-verify\-metamanifest = yes|no +.B sync\-rsync\-verify\-metamanifest = yes|no|true|false Require the repository to contain a signed MetaManifest and verify -it using \fBapp\-portage/gemato\fR. Defaults to no. +it using \fBapp\-portage/gemato\fR. Defaults to no, false. .TP -.B sync\-webrsync\-delta = true|false +.B sync\-webrsync\-delta = yes|no|true|false Use \fBapp\-portage/emerge\-delta\-webrsync\fR to minimize bandwidth. -Defaults to false. +Defaults to no, false. .TP -.B sync\-webrsync\-keep\-snapshots = true|false -Keep snapshots in \fBDISTDIR\fR (do not delete). Defaults to false. +.B sync\-webrsync\-keep\-snapshots = yes|no|true|false +Keep snapshots in \fBDISTDIR\fR (do not delete). Defaults to no, false. .TP -.B sync\-webrsync\-verify\-signature = true|false +.B sync\-webrsync\-verify\-signature = yes|no|true|false Require the detached tarball signature to contain a good OpenPGP signature. This uses the OpenPGP key(ring) specified by the -sync\-openpgp\-key\-path setting. Defaults to false. +sync\-openpgp\-key\-path setting. Defaults to no, false. .RE diff --git a/pylintrc b/pylintrc new file mode 100644 index 000000000..19f59c165 --- /dev/null +++ b/pylintrc @@ -0,0 +1,464 @@ +[MASTER] + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--/ +# disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +#disable=no-absolute-import,bad-continuation,C0103,C0114,C0115,E1101,W0201,no-name-in-module +disable=all +enable=redefined-builtin,useless-object-inheritance,trailing-newlines + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-whitelist= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=.git + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use. +jobs=0 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# Pickle collected data for later comparisons. +persistent=no + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. +confidence=HIGH + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'error', 'warning', 'refactor', and 'convention' +# which contain the number of messages in each category, as well as 'statement' +# which is the total number of statements analyzed. This score is used by the +# global evaluation report (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + e, + _ + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. +#variable-rgx= + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )?<?https?://\S+>?$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string='\t' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module. +max-module-lines=10000 + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma, + dict-separator + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[LOGGING] + +# Format style used to check logging format string. `old` means using % +# formatting, `new` is for `{}` formatting,and `fstr` is for f-strings. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it work, +# install the python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[STRING] + +# This flag controls whether the implicit-str-concat-in-sequence should +# generate a warning on implicit string concatenation in sequences defined over +# several lines. +check-str-concat-over-line-jumps=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=cls + + +[DESIGN] + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=0 + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules=optparse,tkinter.tix + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled). +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled). +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "BaseException, Exception". +overgeneral-exceptions=BaseException diff --git a/repoman/RELEASE-NOTES b/repoman/RELEASE-NOTES index 80541fa5c..0c79956a0 100644 --- a/repoman/RELEASE-NOTES +++ b/repoman/RELEASE-NOTES @@ -1,6 +1,11 @@ Release Notes; upgrade information mainly. Features/major bugfixes are listed in NEWS +repoman-2.3.23 +================================== +* Bug Fixes: + - Bug 637824 Deprecate netsurf.eclass + repoman-2.3.22 ================================== * Bug Fixes: diff --git a/repoman/lib/repoman/__init__.py b/repoman/lib/repoman/__init__.py index 4f3e59e50..fda942c92 100644 --- a/repoman/lib/repoman/__init__.py +++ b/repoman/lib/repoman/__init__.py @@ -20,9 +20,6 @@ except ImportError as e: sys.stderr.write(" "+str(e)+"\n\n") raise -if sys.hexversion >= 0x3000000: - # pylint: disable=W0622 - long = int VERSION = "HEAD" @@ -62,10 +59,10 @@ if VERSION == 'HEAD': head_timestamp = None if len(output_lines) > 3: try: - head_timestamp = long(output_lines[3]) + head_timestamp = int(output_lines[3]) except ValueError: pass - timestamp = long(time.time()) + timestamp = int(time.time()) if head_timestamp is not None and timestamp > head_timestamp: timestamp = timestamp - head_timestamp if not patchlevel: diff --git a/repoman/lib/repoman/actions.py b/repoman/lib/repoman/actions.py index 387ba7800..1afc4acb1 100644 --- a/repoman/lib/repoman/actions.py +++ b/repoman/lib/repoman/actions.py @@ -2,7 +2,7 @@ # Copyright 1999-2019 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function, unicode_literals +from __future__ import print_function import errno import io @@ -41,7 +41,7 @@ from repoman import VERSION bad = create_color_func("BAD") -class Actions(object): +class Actions: '''Handles post check result output and performs the various vcs activities for committing the results''' diff --git a/repoman/lib/repoman/copyrights.py b/repoman/lib/repoman/copyrights.py index 275dcbc3f..9e5a32e34 100644 --- a/repoman/lib/repoman/copyrights.py +++ b/repoman/lib/repoman/copyrights.py @@ -20,7 +20,7 @@ _copyright_re2 = \ re.compile(br'^(# Copyright )(\d\d\d\d)( Gentoo (Foundation|Authors))\b') -class _copyright_repl(object): +class _copyright_repl: __slots__ = ('year',) def __init__(self, year): diff --git a/repoman/lib/repoman/errors.py b/repoman/lib/repoman/errors.py index 9cf113ba0..3509ecc06 100644 --- a/repoman/lib/repoman/errors.py +++ b/repoman/lib/repoman/errors.py @@ -1,6 +1,6 @@ # -*- coding:utf-8 -*- -from __future__ import print_function, unicode_literals +from __future__ import print_function import sys diff --git a/repoman/lib/repoman/gpg.py b/repoman/lib/repoman/gpg.py index 7dac46f41..4e55a6e37 100644 --- a/repoman/lib/repoman/gpg.py +++ b/repoman/lib/repoman/gpg.py @@ -1,6 +1,6 @@ # -*- coding:utf-8 -*- -from __future__ import print_function, unicode_literals +from __future__ import print_function import errno import logging diff --git a/repoman/lib/repoman/main.py b/repoman/lib/repoman/main.py index 731e8eae2..38c6a1759 100755 --- a/repoman/lib/repoman/main.py +++ b/repoman/lib/repoman/main.py @@ -3,7 +3,7 @@ # Copyright 1999-2017 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function, unicode_literals +from __future__ import print_function import io import logging @@ -36,8 +36,6 @@ from repoman import utilities from repoman.modules.vcs.settings import VCSSettings from repoman import VERSION -if sys.hexversion >= 0x3000000: - basestring = str bad = create_color_func("BAD") diff --git a/repoman/lib/repoman/metadata.py b/repoman/lib/repoman/metadata.py index 4537d2ce2..2aaedd4b0 100644 --- a/repoman/lib/repoman/metadata.py +++ b/repoman/lib/repoman/metadata.py @@ -1,6 +1,6 @@ # -*- coding:utf-8 -*- -from __future__ import print_function, unicode_literals +from __future__ import print_function import errno import logging @@ -14,11 +14,6 @@ from portage import os from portage.output import green from portage.package.ebuild.fetch import fetch -if sys.hexversion >= 0x3000000: - basestring = str - -if sys.hexversion >= 0x3000000: - basestring = str # Note: This URI is hardcoded in all metadata.xml files. We can't # change it without updating all the xml files in the tree. diff --git a/repoman/lib/repoman/modules/commit/manifest.py b/repoman/lib/repoman/modules/commit/manifest.py index 573710a62..e71c1abcb 100644 --- a/repoman/lib/repoman/modules/commit/manifest.py +++ b/repoman/lib/repoman/modules/commit/manifest.py @@ -11,7 +11,7 @@ from portage.package.ebuild.digestgen import digestgen from portage.util import writemsg_level -class Manifest(object): +class Manifest: '''Creates as well as checks pkg Manifest entries/files''' def __init__(self, **kwargs): diff --git a/repoman/lib/repoman/modules/commit/repochecks.py b/repoman/lib/repoman/modules/commit/repochecks.py index bedbdaf34..523d414d2 100644 --- a/repoman/lib/repoman/modules/commit/repochecks.py +++ b/repoman/lib/repoman/modules/commit/repochecks.py @@ -1,6 +1,6 @@ # -*- coding:utf-8 -*- -from __future__ import print_function, unicode_literals +from __future__ import print_function from portage.output import red diff --git a/repoman/lib/repoman/modules/linechecks/base.py b/repoman/lib/repoman/modules/linechecks/base.py index 39d7ebd78..bcd86773b 100644 --- a/repoman/lib/repoman/modules/linechecks/base.py +++ b/repoman/lib/repoman/modules/linechecks/base.py @@ -3,7 +3,7 @@ import logging import re -class LineCheck(object): +class LineCheck: """Run a check on a line of an ebuild.""" """A regular expression to determine whether to ignore the line""" ignore_line = False diff --git a/repoman/lib/repoman/modules/linechecks/config.py b/repoman/lib/repoman/modules/linechecks/config.py index 6e4c5314e..69cf8e8aa 100644 --- a/repoman/lib/repoman/modules/linechecks/config.py +++ b/repoman/lib/repoman/modules/linechecks/config.py @@ -6,8 +6,6 @@ """This module contains functions used in Repoman to ascertain the quality and correctness of an ebuild.""" -from __future__ import unicode_literals - import collections import logging import os @@ -37,7 +35,7 @@ def merge(dict1, dict2): return result -class LineChecksConfig(object): +class LineChecksConfig: '''Holds our LineChecks configuration data and operation functions''' def __init__(self, repo_settings): diff --git a/repoman/lib/repoman/modules/linechecks/controller.py b/repoman/lib/repoman/modules/linechecks/controller.py index a3dfd9bd1..50ca5229f 100644 --- a/repoman/lib/repoman/modules/linechecks/controller.py +++ b/repoman/lib/repoman/modules/linechecks/controller.py @@ -18,7 +18,7 @@ MODULES_PATH = os.path.dirname(__file__) logging.debug("LineChecks module path: %s", MODULES_PATH) -class LineCheckController(object): +class LineCheckController: '''Initializes and runs the LineCheck checks''' def __init__(self, repo_settings, linechecks): diff --git a/repoman/lib/repoman/modules/linechecks/deprecated/inherit.py b/repoman/lib/repoman/modules/linechecks/deprecated/inherit.py index f307d46fd..60410347b 100644 --- a/repoman/lib/repoman/modules/linechecks/deprecated/inherit.py +++ b/repoman/lib/repoman/modules/linechecks/deprecated/inherit.py @@ -17,6 +17,7 @@ class InheritDeprecated(LineCheck): "bash-completion": "bash-completion-r1", "boost-utils": False, "clutter": "gnome2", + "cmake-utils": "cmake", "confutils": False, "distutils": "distutils-r1", "epatch": "(eapply since EAPI 6)", @@ -32,6 +33,7 @@ class InheritDeprecated(LineCheck): "gst-plugins10": "gstreamer", "ltprune": False, "mono": "mono-env", + "netsurf": False, "python": "python-r1 / python-single-r1 / python-any-r1", "ruby": "ruby-ng", "user": "GLEP 81", diff --git a/repoman/lib/repoman/modules/scan/ebuild/ebuild.py b/repoman/lib/repoman/modules/scan/ebuild/ebuild.py index 70011e387..14d6178d7 100644 --- a/repoman/lib/repoman/modules/scan/ebuild/ebuild.py +++ b/repoman/lib/repoman/modules/scan/ebuild/ebuild.py @@ -1,6 +1,6 @@ # -*- coding:utf-8 -*- -from __future__ import print_function, unicode_literals +from __future__ import print_function import re import stat diff --git a/repoman/lib/repoman/modules/scan/metadata/ebuild_metadata.py b/repoman/lib/repoman/modules/scan/metadata/ebuild_metadata.py index 2edf8f7f2..6c0822897 100644 --- a/repoman/lib/repoman/modules/scan/metadata/ebuild_metadata.py +++ b/repoman/lib/repoman/modules/scan/metadata/ebuild_metadata.py @@ -5,9 +5,6 @@ import re import sys -if sys.hexversion >= 0x3000000: - basestring = str - from repoman.modules.scan.scanbase import ScanBase from portage.dep import use_reduce @@ -24,7 +21,7 @@ class EbuildMetadata(ScanBase): def invalidchar(self, **kwargs): ebuild = kwargs.get('ebuild').get() for k, v in ebuild.metadata.items(): - if not isinstance(v, basestring): + if not isinstance(v, str): continue m = NON_ASCII_RE.search(v) if m is not None: diff --git a/repoman/lib/repoman/modules/scan/metadata/use_flags.py b/repoman/lib/repoman/modules/scan/metadata/use_flags.py index 1738fd23e..d9ce8bee9 100644 --- a/repoman/lib/repoman/modules/scan/metadata/use_flags.py +++ b/repoman/lib/repoman/modules/scan/metadata/use_flags.py @@ -11,7 +11,7 @@ from portage import eapi from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use -class USEFlagChecks(object): +class USEFlagChecks: '''Performs checks on USE flags listed in the ebuilds and metadata.xml''' def __init__(self, **kwargs): diff --git a/repoman/lib/repoman/modules/scan/module.py b/repoman/lib/repoman/modules/scan/module.py index baed68989..3321cb224 100644 --- a/repoman/lib/repoman/modules/scan/module.py +++ b/repoman/lib/repoman/modules/scan/module.py @@ -19,7 +19,7 @@ MODULES_PATH = os.path.dirname(__file__) logging.debug("module path: %s", MODULES_PATH) -class ModuleConfig(object): +class ModuleConfig: '''Holds the scan modules configuration information and creates the ordered list of modulles to run''' diff --git a/repoman/lib/repoman/modules/scan/scanbase.py b/repoman/lib/repoman/modules/scan/scanbase.py index aea1bb121..8d7b46336 100644 --- a/repoman/lib/repoman/modules/scan/scanbase.py +++ b/repoman/lib/repoman/modules/scan/scanbase.py @@ -1,7 +1,7 @@ # -*- coding:utf-8 -*- -class ScanBase(object): +class ScanBase: '''Skeleton class for performing a scan for one or more items to check in a pkg directory or ebuild.''' diff --git a/repoman/lib/repoman/modules/vcs/None/status.py b/repoman/lib/repoman/modules/vcs/None/status.py index d6e5ca0e4..bd318b4a8 100644 --- a/repoman/lib/repoman/modules/vcs/None/status.py +++ b/repoman/lib/repoman/modules/vcs/None/status.py @@ -3,7 +3,7 @@ None (non-VCS) module Status class submodule ''' -class Status(object): +class Status: '''Performs status checks on the svn repository''' def __init__(self, qatracker, eadded): diff --git a/repoman/lib/repoman/modules/vcs/bzr/status.py b/repoman/lib/repoman/modules/vcs/bzr/status.py index 199e7f399..f4a1a73e2 100644 --- a/repoman/lib/repoman/modules/vcs/bzr/status.py +++ b/repoman/lib/repoman/modules/vcs/bzr/status.py @@ -7,7 +7,7 @@ from portage import os from repoman._subprocess import repoman_popen -class Status(object): +class Status: '''Performs status checks on the svn repository''' def __init__(self, qatracker, eadded): diff --git a/repoman/lib/repoman/modules/vcs/changes.py b/repoman/lib/repoman/modules/vcs/changes.py index aa4923f8f..3a19ae146 100644 --- a/repoman/lib/repoman/modules/vcs/changes.py +++ b/repoman/lib/repoman/modules/vcs/changes.py @@ -13,7 +13,7 @@ from portage import _unicode_encode from portage.process import spawn -class ChangesBase(object): +class ChangesBase: '''Base Class object to scan and hold the resultant data for all changes to process. ''' diff --git a/repoman/lib/repoman/modules/vcs/cvs/status.py b/repoman/lib/repoman/modules/vcs/cvs/status.py index 3ec88f125..13c8b3450 100644 --- a/repoman/lib/repoman/modules/vcs/cvs/status.py +++ b/repoman/lib/repoman/modules/vcs/cvs/status.py @@ -13,7 +13,7 @@ from portage.output import red, green from portage import _unicode_encode, _unicode_decode -class Status(object): +class Status: '''Performs status checks on the svn repository''' def __init__(self, qatracker, eadded): diff --git a/repoman/lib/repoman/modules/vcs/git/status.py b/repoman/lib/repoman/modules/vcs/git/status.py index e5aa9c741..374cb69c9 100644 --- a/repoman/lib/repoman/modules/vcs/git/status.py +++ b/repoman/lib/repoman/modules/vcs/git/status.py @@ -9,7 +9,7 @@ from portage import os from repoman._subprocess import repoman_popen, repoman_getstatusoutput -class Status(object): +class Status: '''Performs status checks on the git repository''' def __init__(self, qatracker, eadded): diff --git a/repoman/lib/repoman/modules/vcs/hg/status.py b/repoman/lib/repoman/modules/vcs/hg/status.py index 8443554f5..2f9f0ba76 100644 --- a/repoman/lib/repoman/modules/vcs/hg/status.py +++ b/repoman/lib/repoman/modules/vcs/hg/status.py @@ -7,7 +7,7 @@ from portage import os from repoman._subprocess import repoman_popen -class Status(object): +class Status: '''Performs status checks on the svn repository''' def __init__(self, qatracker, eadded): diff --git a/repoman/lib/repoman/modules/vcs/settings.py b/repoman/lib/repoman/modules/vcs/settings.py index a8e91dd27..f90f05963 100644 --- a/repoman/lib/repoman/modules/vcs/settings.py +++ b/repoman/lib/repoman/modules/vcs/settings.py @@ -2,7 +2,7 @@ Repoman VCSSettings modules ''' -from __future__ import print_function, unicode_literals +from __future__ import print_function import logging import sys @@ -13,7 +13,7 @@ from repoman.modules.vcs.vcs import FindVCS from repoman.qa_tracker import QATracker -class VCSSettings(object): +class VCSSettings: '''Holds various VCS settings''' def __init__(self, options=None, repoman_settings=None, repo_settings=None): diff --git a/repoman/lib/repoman/modules/vcs/svn/status.py b/repoman/lib/repoman/modules/vcs/svn/status.py index 6575fe0b0..568670a4b 100644 --- a/repoman/lib/repoman/modules/vcs/svn/status.py +++ b/repoman/lib/repoman/modules/vcs/svn/status.py @@ -15,7 +15,7 @@ from portage import _unicode_encode, _unicode_decode from repoman._subprocess import repoman_popen -class Status(object): +class Status: '''Performs status checks on the svn repository''' def __init__(self, qatracker, eadded): diff --git a/repoman/lib/repoman/modules/vcs/vcs.py b/repoman/lib/repoman/modules/vcs/vcs.py index b4ff3dc9c..b19f486a8 100644 --- a/repoman/lib/repoman/modules/vcs/vcs.py +++ b/repoman/lib/repoman/modules/vcs/vcs.py @@ -1,6 +1,6 @@ # -*- coding:utf-8 -*- -from __future__ import print_function, unicode_literals +from __future__ import print_function import collections import logging diff --git a/repoman/lib/repoman/profile.py b/repoman/lib/repoman/profile.py index 50da91728..b17389c55 100644 --- a/repoman/lib/repoman/profile.py +++ b/repoman/lib/repoman/profile.py @@ -1,13 +1,13 @@ # -*- coding:utf-8 -*- -from __future__ import print_function, unicode_literals +from __future__ import print_function from portage import normalize_path from portage import os from portage.output import red -class ProfileDesc(object): +class ProfileDesc: __slots__ = ('abs_path', 'arch', 'status', 'sub_path', 'tree_path',) def __init__(self, arch, status, sub_path, tree_path): diff --git a/repoman/lib/repoman/qa_data.py b/repoman/lib/repoman/qa_data.py index 01141a617..afb403d8d 100644 --- a/repoman/lib/repoman/qa_data.py +++ b/repoman/lib/repoman/qa_data.py @@ -11,7 +11,7 @@ from repoman._portage import portage from repoman.config import load_config -class QAData(object): +class QAData: def __init__(self): # Create the main exported data variables diff --git a/repoman/lib/repoman/qa_tracker.py b/repoman/lib/repoman/qa_tracker.py index faaf8e398..12e1b88bc 100644 --- a/repoman/lib/repoman/qa_tracker.py +++ b/repoman/lib/repoman/qa_tracker.py @@ -3,7 +3,7 @@ import logging import sys -class QATracker(object): +class QATracker: '''Track all occurrances of Q/A problems detected''' def __init__(self, qacats=None, qawarnings=None): diff --git a/repoman/lib/repoman/repos.py b/repoman/lib/repoman/repos.py index 9c820b08c..515c8cbc6 100644 --- a/repoman/lib/repoman/repos.py +++ b/repoman/lib/repoman/repos.py @@ -22,7 +22,7 @@ GPG_KEY_ID_REGEX = r'(0x)?([0-9a-fA-F]{8}){1,5}!?' bad = portage.output.create_color_func("BAD") -class RepoSettings(object): +class RepoSettings: '''Holds our repo specific settings''' def __init__( diff --git a/repoman/lib/repoman/scanner.py b/repoman/lib/repoman/scanner.py index 5db54bb97..8d2ce9052 100644 --- a/repoman/lib/repoman/scanner.py +++ b/repoman/lib/repoman/scanner.py @@ -1,6 +1,6 @@ # -*- coding:utf-8 -*- -from __future__ import print_function, unicode_literals +from __future__ import print_function import logging from itertools import chain @@ -24,7 +24,7 @@ from repoman.modules.vcs.vcs import vcs_files_to_cps DATA_TYPES = {'dict': dict, 'Future': ExtendedFuture, 'list': list, 'set': set} -class Scanner(object): +class Scanner: '''Primary scan class. Operates all the small Q/A tests and checks''' def __init__(self, repo_settings, myreporoot, config_root, options, diff --git a/repoman/lib/repoman/utilities.py b/repoman/lib/repoman/utilities.py index 790d5e516..2d25fd309 100644 --- a/repoman/lib/repoman/utilities.py +++ b/repoman/lib/repoman/utilities.py @@ -6,7 +6,7 @@ """This module contains utility functions to help repoman find ebuilds to scan""" -from __future__ import print_function, unicode_literals +from __future__ import print_function __all__ = [ "editor_is_executable", @@ -50,9 +50,6 @@ from repoman.copyrights import update_copyright, update_copyright_year normalize_path = util.normalize_path util.initialize_logger() -if sys.hexversion >= 0x3000000: - basestring = str - def have_profile_dir(path, maxdepth=3, filename="profiles.desc"): """ diff --git a/repoman/setup.py b/repoman/setup.py index c3a7adad9..f2664fae4 100755 --- a/repoman/setup.py +++ b/repoman/setup.py @@ -450,7 +450,7 @@ def get_manpages(): setup( name = 'repoman', - version = '2.3.22', + version = '2.3.23', url = 'https://wiki.gentoo.org/wiki/Project:Portage', author = 'Gentoo Portage Development Team', author_email = 'dev-portage@gentoo.org', @@ -492,7 +492,7 @@ setup( 'Intended Audience :: System Administrators', 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)', 'Operating System :: POSIX', - 'Programming Language :: Python', + 'Programming Language :: Python :: 3', 'Topic :: System :: Installation/Setup' ] ) @@ -23,7 +23,6 @@ import tempfile # These are the versions we fully support and require to pass tests. PYTHON_SUPPORTED_VERSIONS = [ - '2.7', '3.6', '3.7', '3.8' @@ -37,7 +36,7 @@ PYTHON_NICE_VERSIONS = [ EPREFIX = os.environ.get('PORTAGE_OVERRIDE_EPREFIX', '/') -class Colors(object): +class Colors: """Simple object holding color constants.""" _COLORS_YES = ('y', 'yes', 'true') @@ -657,7 +657,7 @@ class build_ext(_build_ext): setup( name = 'portage', - version = '2.3.100', + version = '3.0.1', url = 'https://wiki.gentoo.org/wiki/Project:Portage', author = 'Gentoo Portage Development Team', author_email = 'dev-portage@gentoo.org', @@ -714,7 +714,7 @@ setup( 'Intended Audience :: System Administrators', 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)', 'Operating System :: POSIX', - 'Programming Language :: Python', + 'Programming Language :: Python :: 3', 'Topic :: System :: Installation/Setup' ] ) @@ -1,14 +1,12 @@ [tox] -envlist = py27,py36,py37,py38,py39,pypy3 +envlist = py36,py37,py38,py39,pypy3 skipsdist = True [testenv] deps = pygost pyyaml - py27,py36,py37,py38,py39,pypy3: lxml!=4.2.0 - py27: pyblake2 - py27: pysha3 + py36,py37,py38,py39,pypy3: lxml!=4.2.0 setenv = PYTHONPATH={toxinidir}/lib commands = |