diff options
author | Fabian Groffen <grobian@gentoo.org> | 2014-09-28 19:52:16 +0200 |
---|---|---|
committer | Fabian Groffen <grobian@gentoo.org> | 2014-09-28 19:52:16 +0200 |
commit | 990c5f4896b309fdcaf1dbbb5779177ecfcf6e74 (patch) | |
tree | f54ebb8d5bcbfea1986b9c73eee0ec3c5b390577 | |
parent | install_qa_check_macho: introduce QA_IGNORE_INSTALL_NAME_FILES (diff) | |
parent | Use a single grep call for gcc warning checks (diff) | |
download | portage-990c5f4896b309fdcaf1dbbb5779177ecfcf6e74.tar.gz portage-990c5f4896b309fdcaf1dbbb5779177ecfcf6e74.tar.bz2 portage-990c5f4896b309fdcaf1dbbb5779177ecfcf6e74.zip |
Merge remote-tracking branch 'overlays-gentoo-org/master' into prefix
Conflicts:
bin/ebuild-helpers/emake
bin/misc-functions.sh
bin/portageq
doc/Makefile
pym/_emerge/EbuildBuild.py
pym/portage/const.py
pym/portage/dbapi/vartree.py
pym/portage/package/ebuild/doebuild.py
153 files changed, 4264 insertions, 1856 deletions
diff --git a/.gitignore b/.gitignore index 074bb864d..c2dd5349d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.py[co] __pycache__/ *.class +/build /tags diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..69e1659db --- /dev/null +++ b/.travis.yml @@ -0,0 +1,13 @@ +language: python +python: + - 2.6 + - 2.7 + - 3.2 + - 3.3 + - 3.4 + - pypy +script: + - ./setup.py test + - ./setup.py install --root=/tmp/install-root + - sudo rsync -a /tmp/install-root/. / + - python -b -Wd -m portage.tests.runTests diff --git a/DEVELOPING b/DEVELOPING index 0ffcbd76e..55470d85b 100644 --- a/DEVELOPING +++ b/DEVELOPING @@ -174,6 +174,11 @@ change a lot of unrelated things. This makes it easier to see what parts of the system have actually changed. It also makes it easier to cherry-pick and revert commits. Use your common sense! +When you make a significant change, make sure to update RELEASE-NOTES +for the to-be-released version. Very significant changes should be +mentioned in NEWS as well. See the current entries to these files for +examples of what constitutes significant. + Commit messages --------------- @@ -181,7 +186,18 @@ Commit messages should be in the imperative mood with a capitalised header, optionally followed by a newline and a more detailed explanatory text. The headline should be capped at 70 characters, the detailed text at 72. Prefix the message with the component you touched if this makes -sense. Postfix the message with the bug it fixes, if it does. Example: +sense. Postfix the message with the bug it fixes, if it does. + +Feel free to use the following notes (if applicable): +Signed-off-by: Wrote (a substantial portion of) the patch +Reviewed-by: Reviewed the patch thoroughly +Tested-by: Tested the patch thoroughly +Acked-by: Approved the concept but did not read the patch in detail +(typically used by the maintainer of a specific portion, or a lead) +Suggested-by: Designed the implementation +Requested-by: Reported the bug/made the feature request + +Example: " emerge: Fix --tree output (bug 555555) @@ -189,6 +205,10 @@ emerge: Fix --tree output (bug 555555) Make sure newlines appear where they are supposed to. Fix a bug with colourisation of --tree output when used in tandem with --verbose --pretend --ask. + +Signed-off-by: Foo Bar <fbar@gentoo.org> +Reviewed-by: Fu Baz <fub@gentoo.org> +Reported-by: Qux Quux <qq@gentoo.org> " For a more detailed explanation (and rationalisation) of these rules: diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..d65c874a3 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,18 @@ +# docs +include DEVELOPING +include LICENSE +include TEST-NOTES + +# docbook sources +include doc/custom.xsl +recursive-include doc *.docbook + +# extra conf files used in ebuild +include cnf/make.conf.example.* + +# extra files for tests +include .portage_not_installed +include cnf/metadata.dtd + +# extra scripts +include misc/* diff --git a/Makefile b/Makefile deleted file mode 100644 index 92ea19562..000000000 --- a/Makefile +++ /dev/null @@ -1,215 +0,0 @@ -SHELL = /bin/sh -PN ?= portage -PF ?= portage -HOMEPAGE ?= http://www.gentoo.org/proj/en/portage/index.xml -PWD ?= $(shell pwd) -S ?= $(PWD) -WORKDIR ?= $(PWD) -DESTDIR = $(PWD)/image/ -srcdir = $(S) -prefix = /usr -sysconfdir = /etc -exec_prefix = $(prefix) -bindir = $(exec_prefix)/bin -sbindir = $(exec_prefix)/sbin -libdir = $(exec_prefix)/lib -datarootdir = $(prefix)/share -datadir = $(datarootdir) -mandir = $(datarootdir)/man -docdir = $(datarootdir)/doc/$(PF) -htmldir = $(docdir)/html -portage_datadir = $(datarootdir)/$(PN) -portage_confdir = $(portage_datadir)/config -portage_setsdir = $(portage_confdir)/sets -portage_base = $(libdir)/$(PN) -EPYDOC_OPTS = -qqqqq --no-frames --show-imports -INSMODE = 0644 -EXEMODE = 0755 -DIRMODE = 0755 -SYSCONFDIR_FILES = etc-update.conf dispatch-conf.conf -PORTAGE_CONFDIR_FILES = make.conf.example make.globals repos.conf -LOGROTATE_FILES = elog-save-summary -BINDIR_FILES = ebuild egencache emerge emerge-webrsync \ - emirrordist portageq quickpkg repoman -SBINDIR_FILES = archive-conf dispatch-conf emaint \ - env-update etc-update fixpackages regenworld -DOCS = ChangeLog NEWS RELEASE-NOTES -LINGUAS ?= $(shell cd "$(srcdir)/man" && find -mindepth 1 -type d) - -ifdef PYTHONPATH - PYTHONPATH := $(srcdir)/pym:$(PYTHONPATH) -else - PYTHONPATH := $(srcdir)/pym -endif - -all: docbook epydoc - -docbook: - set -e; \ - touch "$(srcdir)/doc/fragment/date"; \ - $(MAKE) -C "$(srcdir)/doc" xhtml xhtml-nochunks - -epydoc: - set -e; \ - env PYTHONPATH="$(PYTHONPATH)" epydoc \ - -o "$(WORKDIR)/epydoc" \ - --name $(PN) \ - --url "$(HOMEPAGE)" \ - $(EPYDOC_OPTS) \ - $$(cd "$(srcdir)" && find pym -name '*.py' | sed \ - -e s:/__init__.py$$:: \ - -e s:\.py$$:: \ - -e s:^pym/:: \ - -e s:/:.:g \ - | sort); \ - rm -f "$(WORKDIR)/epydoc/api-objects.txt"; \ - -test: - set -e; \ - "$(srcdir)/pym/portage/tests/runTests"; \ - -install: - set -e; \ - cd "$(srcdir)/cnf"; \ - install -d -m$(DIRMODE) "$(DESTDIR)$(sysconfdir)"; \ - install -m$(INSMODE) $(SYSCONFDIR_FILES) "$(DESTDIR)$(sysconfdir)"; \ - \ - install -d -m$(DIRMODE) "$(DESTDIR)$(portage_confdir)"; \ - cd "$(srcdir)/cnf"; \ - install -m$(INSMODE) $(PORTAGE_CONFDIR_FILES) \ - "$(DESTDIR)$(portage_confdir)"; \ - install -d -m$(DIRMODE) "$(DESTDIR)$(portage_setsdir)"; \ - cd "$(S)/cnf/sets"; \ - install -m$(INSMODE) *.conf "$(DESTDIR)$(portage_setsdir)"; \ - \ - install -d -m$(DIRMODE) "$(DESTDIR)$(sysconfdir)/logrotate.d"; \ - cd "$(srcdir)/cnf/logrotate.d"; \ - install -m$(INSMODE) $(LOGROTATE_FILES) \ - "$(DESTDIR)$(sysconfdir)/logrotate.d"; \ - \ - for x in $$(cd "$(srcdir)" && find bin -type d) ; do \ - cd "$(srcdir)/$$x"; \ - install -d -m$(DIRMODE) "$(DESTDIR)$(portage_base)/$$x"; \ - files=$$(find . -mindepth 1 -maxdepth 1 -type f ! -type l); \ - if [ -n "$$files" ] ; then \ - install -m$(EXEMODE) $$files \ - "$(DESTDIR)$(portage_base)/$$x"; \ - fi; \ - symlinks=$$(find . -mindepth 1 -maxdepth 1 -type l); \ - if [ -n "$$symlinks" ] ; then \ - cp -P $$symlinks "$(DESTDIR)$(portage_base)/$$x"; \ - fi; \ - done; \ - \ - for x in $$(cd "$(srcdir)" && find pym/* -type d \ - ! -path "pym/portage/tests*") ; do \ - cd "$(srcdir)/$$x"; \ - files=$$(echo *.py); \ - if [ -z "$$files" ] || [ "$$files" = "*.py" ]; then \ - # __pycache__ directories contain no py files \ - continue; \ - fi; \ - install -d -m$(DIRMODE) "$(DESTDIR)$(portage_base)/$$x"; \ - install -m$(INSMODE) $$files "$(DESTDIR)$(portage_base)/$$x"; \ - done; \ - \ - install -d -m$(DIRMODE) "$(DESTDIR)$(bindir)"; \ - relative_path=".."; \ - x=$(bindir) ; \ - y="$(portage_base)"; \ - if [ "$${x#$(prefix)}" != "$$x" ] && \ - [ "$${y#$(prefix)}" != "$$y" ]; then \ - x=$${x#$(prefix)}; \ - y=$${y#$(prefix)}; \ - fi; \ - x=$${x%/*}; \ - while [ -n "$$x" ] ; do \ - relative_path=$${relative_path}/..; \ - x=$${x%/*}; \ - done; \ - relative_path=$$relative_path$$y; \ - for x in $(BINDIR_FILES) ; do \ - ln -sf "$$relative_path/bin/$$x" \ - "$(DESTDIR)$(bindir)/$$x"; \ - done; \ - \ - install -d -m$(DIRMODE) "$(DESTDIR)$(sbindir)"; \ - relative_path=".."; \ - x=$(sbindir) ; \ - y="$(portage_base)"; \ - if [ "$${x#$(prefix)}" != "$$x" ] && \ - [ "$${y#$(prefix)}" != "$$y" ]; then \ - x=$${x#$(prefix)}; \ - y=$${y#$(prefix)}; \ - fi; \ - x=$${x%/*}; \ - while [ -n "$$x" ] ; do \ - relative_path=$${relative_path}/..; \ - x=$${x%/*}; \ - done; \ - relative_path=$$relative_path$$y; \ - for x in $(SBINDIR_FILES) ; do \ - ln -sf "$$relative_path/bin/$$x" \ - "$(DESTDIR)$(sbindir)/$$x"; \ - done; \ - \ - ln -sf "$$relative_path/bin/env-update" \ - "$(DESTDIR)$(sbindir)/update-env"; \ - ln -sf "$$relative_path/bin/etc-update" \ - "$(DESTDIR)$(sbindir)/update-etc"; \ - \ - # We install some minimal tests for use as a preinst sanity check. \ - # These tests must be able to run without a full source tree and \ - # without relying on a previous portage instance being installed. \ - install -d -m$(DIRMODE) \ - "$(DESTDIR)$(portage_base)/pym/portage/tests"; \ - install -m$(EXEMODE) "$(srcdir)/pym/portage/tests/runTests" \ - "$(DESTDIR)$(portage_base)/pym/portage/tests"; \ - cd "$(srcdir)/pym/portage/tests"; \ - install -m$(INSMODE) *.py \ - "$(DESTDIR)$(portage_base)/pym/portage/tests"; \ - install -d -m$(DIRMODE) \ - "$(DESTDIR)$(portage_base)/pym/portage/tests/lint"; \ - cd "$(srcdir)/pym/portage/tests/lint"; \ - install -m$(INSMODE) *.py __test__ \ - "$(DESTDIR)$(portage_base)/pym/portage/tests/lint"; \ - \ - install -d -m$(DIRMODE) "$(DESTDIR)$(docdir)"; \ - cd "$(srcdir)"; \ - install -m $(INSMODE) $(DOCS) "$(DESTDIR)$(docdir)"; \ - \ - for x in "" $(LINGUAS); do \ - for y in 1 5 ; do \ - if [ -d "$(srcdir)/man/$$x" ]; then \ - cd "$(srcdir)/man/$$x"; \ - files=$$(echo *.$$y); \ - if [ -z "$$files" ] || [ "$$files" = "*.$$y" ]; then \ - continue; \ - fi; \ - install -d -m$(DIRMODE) "$(DESTDIR)$(mandir)/$$x/man$$y"; \ - install -m$(INSMODE) *.$$y "$(DESTDIR)$(mandir)/$$x/man$$y"; \ - fi; \ - done; \ - done; \ - \ - if [ -f "$(srcdir)/doc/portage.html" ] ; then \ - install -d -m$(DIRMODE) "$(DESTDIR)$(htmldir)"; \ - cd "$(srcdir)/doc"; \ - install -m$(INSMODE) *.html "$(DESTDIR)$(htmldir)"; \ - fi; \ - \ - if [ -d "$(WORKDIR)/epydoc" ] ; then \ - install -d -m$(DIRMODE) "$(DESTDIR)$(htmldir)"; \ - cp -pPR "$(WORKDIR)/epydoc" \ - "$(DESTDIR)$(htmldir)/api"; \ - cd "$(DESTDIR)$(htmldir)/api"; \ - find . -type d | xargs chmod $(DIRMODE); \ - find . -type f | xargs chmod $(INSMODE); \ - fi; \ - -clean: - set -e; \ - $(MAKE) -C "$(srcdir)/doc" clean; \ - rm -rf "$(WORKDIR)/epydoc"; \ - -.PHONY: all clean docbook epydoc install test @@ -1,5 +1,22 @@ News (mainly features/major bug fixes) + +portage-2.2.13 +------------- +* New setup.py build system and distutils-r1 ebuild at EAPI 5 +* PORTAGE_{BIN,PYM}_PATH variables. If you want to call internal portage tools, + you need to query PORTAGE_BIN_PATH and PORTAGE_PYM_PATH, use them + and re-export them. +* New @changed-deps set to update packages which need dependecy change updates. + + +portage-2.2.12 +------------- +* New emaint module "merges" for finding and fixing failed merges + If a pkg fails to merge to the live filesystem correctly, that pkg may + not work correctly if at all. This module scans the installed pkg database + for those failures and can re-emerge those packages. + portage-2.2.1 ------------- @@ -160,7 +177,7 @@ portage-2.1.3 RDEPEND. This makes PDEPEND more useful for breaking the dependency cycles that are sometimes triggered by RDEPEND and DEPEND. * Enable "echo" and "save_summary" elog modules by default -* Fix -* handling in package.keywords to work as intended (reset the accepted +* Fix -* handling in package.keywords to work as intended (reset the accepted keywords list), also see RELEASE-NOTES. * Make elog functionality available to python code @@ -183,9 +200,9 @@ portage-2.1.2 order and detection of circular dependencies. * The world and system sets allow automatic update of all installed slots. * DEPEND atoms support SLOT dependencies of the form ${CATEGORY}/${PN}:${SLOT}. -* Development: Extend PYTHONPATH support to allow overriding the hardcoded +* Development: Extend PYTHONPATH support to allow overriding the hardcoded /usr/lib/portage/pym for development/testing purposes -* New "finalize" hook for elog modules enabling modules to perform actions on +* New "finalize" hook for elog modules enabling modules to perform actions on shutdown (like dispatching collected messages). * New elog modules mail_summary and save_summary that act like the mail/save modules except that they merge messages from multiple packages in a single @@ -203,7 +220,7 @@ portage-2.1.1 portage-2.1 (ongoing via pre releases) ------------ -* FEATURES="confcache" integration; global caching to speed up configure calls, +* FEATURES="confcache" integration; global caching to speed up configure calls, requires dev-util/confcache * elog framework and accompanying modules for logging ebuild warnings, errors and general notices. Collects eerror/ewarn/elog/einfo messages. @@ -213,7 +230,7 @@ portage-2.1 (ongoing via pre releases) * config files as directories enabling more flexible settings management. * Addition of an register_die_hook method that allows ebuild/eclasses to register functions to be called for better debugging on errors. -* Addition of pre and post user hookable functions for each ebuild phase, accessible +* Addition of pre and post user hookable functions for each ebuild phase, accessible via portage bashrc. Example would be pre_src_unpack . * cache refactoring- runtime improvement from 35% -> 65%. * Intelligently handle and display USE_EXPAND-based IUSE variables. diff --git a/RELEASE-NOTES b/RELEASE-NOTES index 44cb1c3c7..7a46612d5 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -1,6 +1,54 @@ Release Notes; upgrade information mainly. Features/major bugfixes are listed in NEWS +portage-2.2.13 +================================== +* Bug Fixes: + - Bug # 438976 Remove DESCRIPTION.punctuation check from repoman + - Bug # 520542 Replace .append() with .add() for set variables + - Bug # 515230 package_tracker.match: account for unevaluated_atom + - Bug # 508762 _slot_operator_update_probe: This fixes the + check_reverse_dependencies function. + - Bug # 522362 Fix config.setcpv to regenerate USE settings in order to + account for package.env USE settings from the previous package + instance. + - Bug # 507482 Run distcc-pump server throughout src_configure() + to src_install() + - Bug # 520950 This handles a case which occurs when + _solve_non_slot_operator_slot_conflicts calls _create_graph. + - Bug # 520752 Make email date, timestamp RFC-compliant + - Bug # 520378 Fix hard-coded emerge-fetch.log locations + - Bug # 522652 For cases such as || ( X <A-2 ), where X is unsatisfiable + and A-1 is installed, fix dep_zapdeps to make the correct choice. + - Bug # 510270 This fixes an IndexError in + _solve_non_slot_operator_slot_conflicts which occurs when none of the + conflict packages matched a particular atom. +* No longer include a Changelog with release tarball. + For a complete log of the Changes please refer to the git log viewable + online at https://github.com/gentoo/portage/commits/master + + +portage-2.2.12 +================================== +* Bug Fixes: + - Bug # 519074 fix invalid locale setting + - Repoman: fix atom.blocker checks + - Bug # 519124 properly decode formatted number for localized_size() + - Revert an incorrect test fix from 2.2.11 which broke mime type detection + - Bug # 518968 Fix and incorrect userquery change + - QA systemd warning check for using /etc/conf.d + - QA Use pngfix to find broken PNG files + - Bug # 512578 Prepend '=' to unmerge atoms + - Repoman: Do not report DESCRIPTION.punctuation warning for "etc." + - Fix a py2/py3 discrepency involing integer division causing number output + to be inconsistent + - Fix an unicode-decode error in a gettaddrinfo() error message +* New emaint module "merges" for finding and fixing failed merges + If a pkg fails to merge to the live filesystem correctly, that pkg may + not work correctly if at all. This module scans the installed pkg database + for those failures and can re-emerge those packages. + + portage-2.2.11 ================================== * Bug Fixes: @@ -8,9 +56,24 @@ portage-2.2.11 - Bug # 505428 RO only filesystem check - Bug # 506186 TaskSequence starting bug. - Sort repoman check output - - remove obsolete repoman eclass checks + - Remove obsolete repoman eclass checks - Bug # 505944 Improve mismatch checking - Bug # 488820 fix @security crash + - Bug # 438976 Add DESCRIPTION.punctuation check to repoman + - Add ruby18 warning for deprecated ruby target to repoman + - Add Python version to Portage version line + - Prevent rebuild code from performing unwanted repository changes + - Include "::repository" more consistently in output + - Make the slot conflict handler output more debug information + - Bug # 487074 Don't split suggested commands when printing them + - Handle 'mkdir -p /etc/portage/make.profile/packages' gracefully + -- i.e. produce an error instead of crashing with a traceback + - Implement --alert + - Bug # 516428 Make repoman warn if non-virtuals depend on + perl-core + - Prefer install-xattr to install.py as a wrapper to coreutils' + /usr/bin/install to preserve file system extended attribute. + portage-2.2.10 ================================== diff --git a/bin/archive-conf b/bin/archive-conf index 599e5693b..b427b8803 100755 --- a/bin/archive-conf +++ b/bin/archive-conf @@ -14,8 +14,8 @@ from __future__ import print_function import sys from os import path as osp -pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") -sys.path.insert(0, pym_path) +if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), ".portage_not_installed")): + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage portage._internal_caller = True diff --git a/bin/binhost-snapshot b/bin/binhost-snapshot index ce156a422..f289394ab 100755 --- a/bin/binhost-snapshot +++ b/bin/binhost-snapshot @@ -13,8 +13,8 @@ except ImportError: from urlparse import urlparse from os import path as osp -pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") -sys.path.insert(0, pym_path) +if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), ".portage_not_installed")): + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage portage._internal_caller = True from portage.util._argparse import ArgumentParser diff --git a/bin/chpathtool.py b/bin/chpathtool.py index 7e9f26042..c4716d97d 100755 --- a/bin/chpathtool.py +++ b/bin/chpathtool.py @@ -2,9 +2,11 @@ # Copyright 2011-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -"""Helper tool for converting installed files to custom prefixes. +doc = """Helper tool for converting installed files to custom prefixes. In other words, eprefixy $D for Gentoo/Prefix.""" +__doc__ = doc + import io import os @@ -152,7 +154,7 @@ def chpath_inplace_symlink(filename, st, old, new): def main(argv): - parser = ArgumentParser(description=__doc__) + parser = ArgumentParser(description=doc) try: parser.add_argument('location', default=None, help='root directory (e.g. $D)') @@ -166,7 +168,7 @@ def main(argv): except OptionError: # Argument parsing compatibility for Python 2.6 using optparse. if sys.hexversion < 0x2070000: - parser = OptionParser(description=__doc__, + parser = OptionParser(description=doc, usage="usage: %prog [-h] location old new\n\n" + \ " location: root directory (e.g. $D)\n" + \ " old: original build prefix (e.g. /)\n" + \ diff --git a/bin/clean_locks b/bin/clean_locks index 73213b52b..d44162075 100755 --- a/bin/clean_locks +++ b/bin/clean_locks @@ -6,8 +6,8 @@ from __future__ import print_function import sys, errno from os import path as osp -pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") -sys.path.insert(0, pym_path) +if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), ".portage_not_installed")): + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage portage._internal_caller = True diff --git a/bin/deprecated-path b/bin/deprecated-path new file mode 100755 index 000000000..b8aaadbda --- /dev/null +++ b/bin/deprecated-path @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# Author: +# + +source /lib/gentoo/functions.sh + +scriptpath=${BASH_SOURCE[0]} +scriptname=${scriptpath##*/} + +IFS=':' + +for path in ${PATH}; do + [[ -x ${path}/${scriptname} ]] || continue + [[ ${path}/${scriptname} -ef ${scriptpath} ]] && continue + + unset IFS + eerror "Deprecation warning: Calling ${scriptname} from wrong path: '${scriptpath}'" + eerror "Correct path should be '${path}/${scriptname}', Please correct your scripts or file a bug with the maintainer..." + exec "${path}/${scriptname}" "$@" +done + +unset IFS + +eerror "ERROR: portage file: deprecated-path: Failed to locate ${scriptname} in PATH" +eerror "PATH: ${PATH}" +exit 1 diff --git a/bin/dispatch-conf b/bin/dispatch-conf index a1fe2061c..6f06b35a0 100755 --- a/bin/dispatch-conf +++ b/bin/dispatch-conf @@ -17,8 +17,8 @@ from stat import ST_GID, ST_MODE, ST_UID from random import random import atexit, re, shutil, stat, sys from os import path as osp -pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") -sys.path.insert(0, pym_path) +if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), ".portage_not_installed")): + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage portage._internal_caller = True from portage import os diff --git a/bin/ebuild b/bin/ebuild index bb0d79a75..aa35bb620 100755 --- a/bin/ebuild +++ b/bin/ebuild @@ -38,8 +38,8 @@ signal.signal(debug_signum, debug_signal) import io import os from os import path as osp -pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") -sys.path.insert(0, pym_path) +if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), ".portage_not_installed")): + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage portage._internal_caller = True from portage import os diff --git a/bin/ebuild-helpers/emake b/bin/ebuild-helpers/emake index 60286ec9d..dcb64a399 100755 --- a/bin/ebuild-helpers/emake +++ b/bin/ebuild-helpers/emake @@ -13,7 +13,7 @@ source "${PORTAGE_BIN_PATH:-@PORTAGE_BASE@/bin}"/isolated-functions.sh if [[ $PORTAGE_QUIET != 1 ]] ; then ( - for arg in ${MAKE:-make} $MAKEOPTS $EXTRA_EMAKE "$@" ; do + for arg in ${MAKE:-make} $MAKEOPTS "$@" $EXTRA_EMAKE ; do [[ ${arg} == *" "* ]] \ && printf "'%s' " "${arg}" \ || printf "%s " "${arg}" @@ -22,7 +22,7 @@ if [[ $PORTAGE_QUIET != 1 ]] ; then ) >&2 fi -${MAKE:-make} SHELL="${BASH:-/bin/bash}" ${MAKEOPTS} ${EXTRA_EMAKE} "$@" +${MAKE:-make} SHELL="${BASH:-/bin/bash}" ${MAKEOPTS} "$@" ${EXTRA_EMAKE} ret=$? [[ $ret -ne 0 ]] && __helpers_die "${0##*/} failed" exit $ret diff --git a/bin/ebuild-helpers/xattr/install b/bin/ebuild-helpers/xattr/install index b1d23159a..233459fcc 100755 --- a/bin/ebuild-helpers/xattr/install +++ b/bin/ebuild-helpers/xattr/install @@ -4,9 +4,32 @@ PORTAGE_BIN_PATH=${PORTAGE_BIN_PATH:-/usr/lib/portage/bin} PORTAGE_PYM_PATH=${PORTAGE_PYM_PATH:-/usr/lib/portage/pym} +INSTALL_XATTR=${EPREFIX}/usr/bin/install-xattr # Use safe cwd, avoiding unsafe import for bug #469338. export __PORTAGE_HELPER_CWD=${PWD} cd "${PORTAGE_PYM_PATH}" export __PORTAGE_HELPER_PATH=${BASH_SOURCE[0]} -PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \ - exec "${PORTAGE_PYTHON:-/usr/bin/python}" "${PORTAGE_BIN_PATH}/install.py" "$@" + +if [[ ${PORTAGE_INSTALL_XATTR_IMPLEMENTATION} == "c" ]]; then + implementation="c" +elif [[ ${PORTAGE_INSTALL_XATTR_IMPLEMENTATION} == "python" ]]; then + implementation="python" +else + # If PORTAGE_INSTALL_XATTR_IMPLEMENTATION is unset or not set to "c" or "python" + # then we'll autodetect, preferring "c" but falling back on "python" + if [[ -x "${INSTALL_XATTR}" ]]; then + implementation="c" + else + implementation="python" + fi +fi + +if [[ "${implementation}" == "c" ]]; then + exec "${INSTALL_XATTR}" "$@" +elif [[ "${implementation}" == "python" ]]; then + PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \ + exec "${PORTAGE_PYTHON:-/usr/bin/python}" "${PORTAGE_BIN_PATH}/install.py" "$@" +else + echo "Unknown implementation for PORTAGE_INSTALL_XATTR_IMPLEMENTATION" + exit -1 +fi diff --git a/bin/ebuild-ipc.py b/bin/ebuild-ipc.py index 1c7290fd3..5a702417e 100755 --- a/bin/ebuild-ipc.py +++ b/bin/ebuild-ipc.py @@ -24,15 +24,21 @@ else: signal.signal(debug_signum, debug_signal) -# Avoid sandbox violations after python upgrade. -pym_path = os.path.join(os.path.dirname( - os.path.dirname(os.path.realpath(__file__))), "pym") +if os.path.isfile(os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), ".portage_not_installed")): + pym_paths = [os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "pym")] + sys.path.insert(0, pym_paths[0]) +else: + import distutils.sysconfig + pym_paths = [os.path.join(distutils.sysconfig.get_python_lib(), x) for x in ("_emerge", "portage")] +# Avoid sandbox violations after Python upgrade. if os.environ.get("SANDBOX_ON") == "1": sandbox_write = os.environ.get("SANDBOX_WRITE", "").split(":") - if pym_path not in sandbox_write: - sandbox_write.append(pym_path) - os.environ["SANDBOX_WRITE"] = \ - ":".join(filter(None, sandbox_write)) + for pym_path in pym_paths: + if pym_path not in sandbox_write: + sandbox_write.append(pym_path) + os.environ["SANDBOX_WRITE"] = ":".join(filter(None, sandbox_write)) + del pym_path, sandbox_write +del pym_paths import portage portage._internal_caller = True diff --git a/bin/ebuild.sh b/bin/ebuild.sh index ef19e5ec7..103caee1b 100755 --- a/bin/ebuild.sh +++ b/bin/ebuild.sh @@ -230,10 +230,10 @@ inherit() { __export_funcs_var=__export_functions_$ECLASS_DEPTH unset $__export_funcs_var - if [ "${EBUILD_PHASE}" != "depend" ] && \ - [ "${EBUILD_PHASE}" != "nofetch" ] && \ - [[ ${EBUILD_PHASE} != *rm ]] && \ - [[ ${EMERGE_FROM} != "binary" ]] ; then + if [[ ${EBUILD_PHASE} != depend && ${EBUILD_PHASE} != nofetch && \ + ${EBUILD_PHASE} != *rm && ${EMERGE_FROM} != "binary" && \ + -z ${_IN_INSTALL_QA_CHECK} ]] + then # This is disabled in the *rm phases because they frequently give # false alarms due to INHERITED in /var/db/pkg being outdated # in comparison the the eclasses from the portage tree. It's @@ -256,70 +256,75 @@ inherit() { debug-print "inherit: $1 -> $location" [[ -z ${location} ]] && die "${1}.eclass could not be found by inherit()" - #We need to back up the values of *DEPEND to B_*DEPEND - #(if set).. and then restore them after the inherit call. - - #turn off glob expansion - set -f - - # Retain the old data and restore it later. - unset B_IUSE B_REQUIRED_USE B_DEPEND B_RDEPEND B_PDEPEND B_HDEPEND - [ "${IUSE+set}" = set ] && B_IUSE="${IUSE}" - [ "${REQUIRED_USE+set}" = set ] && B_REQUIRED_USE="${REQUIRED_USE}" - [ "${DEPEND+set}" = set ] && B_DEPEND="${DEPEND}" - [ "${RDEPEND+set}" = set ] && B_RDEPEND="${RDEPEND}" - [ "${PDEPEND+set}" = set ] && B_PDEPEND="${PDEPEND}" - [ "${HDEPEND+set}" = set ] && B_HDEPEND="${HDEPEND}" - unset IUSE REQUIRED_USE DEPEND RDEPEND PDEPEND HDEPEND - #turn on glob expansion - set +f + # inherits in QA checks can't handle metadata assignments + if [[ -z ${_IN_INSTALL_QA_CHECK} ]]; then + #We need to back up the values of *DEPEND to B_*DEPEND + #(if set).. and then restore them after the inherit call. + + #turn off glob expansion + set -f + + # Retain the old data and restore it later. + unset B_IUSE B_REQUIRED_USE B_DEPEND B_RDEPEND B_PDEPEND B_HDEPEND + [ "${IUSE+set}" = set ] && B_IUSE="${IUSE}" + [ "${REQUIRED_USE+set}" = set ] && B_REQUIRED_USE="${REQUIRED_USE}" + [ "${DEPEND+set}" = set ] && B_DEPEND="${DEPEND}" + [ "${RDEPEND+set}" = set ] && B_RDEPEND="${RDEPEND}" + [ "${PDEPEND+set}" = set ] && B_PDEPEND="${PDEPEND}" + [ "${HDEPEND+set}" = set ] && B_HDEPEND="${HDEPEND}" + unset IUSE REQUIRED_USE DEPEND RDEPEND PDEPEND HDEPEND + #turn on glob expansion + set +f + fi __qa_source "$location" || die "died sourcing $location in inherit()" - #turn off glob expansion - set -f - - # If each var has a value, append it to the global variable E_* to - # be applied after everything is finished. New incremental behavior. - [ "${IUSE+set}" = set ] && E_IUSE+="${E_IUSE:+ }${IUSE}" - [ "${REQUIRED_USE+set}" = set ] && E_REQUIRED_USE+="${E_REQUIRED_USE:+ }${REQUIRED_USE}" - [ "${DEPEND+set}" = set ] && E_DEPEND+="${E_DEPEND:+ }${DEPEND}" - [ "${RDEPEND+set}" = set ] && E_RDEPEND+="${E_RDEPEND:+ }${RDEPEND}" - [ "${PDEPEND+set}" = set ] && E_PDEPEND+="${E_PDEPEND:+ }${PDEPEND}" - [ "${HDEPEND+set}" = set ] && E_HDEPEND+="${E_HDEPEND:+ }${HDEPEND}" - - [ "${B_IUSE+set}" = set ] && IUSE="${B_IUSE}" - [ "${B_IUSE+set}" = set ] || unset IUSE - - [ "${B_REQUIRED_USE+set}" = set ] && REQUIRED_USE="${B_REQUIRED_USE}" - [ "${B_REQUIRED_USE+set}" = set ] || unset REQUIRED_USE - - [ "${B_DEPEND+set}" = set ] && DEPEND="${B_DEPEND}" - [ "${B_DEPEND+set}" = set ] || unset DEPEND - - [ "${B_RDEPEND+set}" = set ] && RDEPEND="${B_RDEPEND}" - [ "${B_RDEPEND+set}" = set ] || unset RDEPEND - - [ "${B_PDEPEND+set}" = set ] && PDEPEND="${B_PDEPEND}" - [ "${B_PDEPEND+set}" = set ] || unset PDEPEND - - [ "${B_HDEPEND+set}" = set ] && HDEPEND="${B_HDEPEND}" - [ "${B_HDEPEND+set}" = set ] || unset HDEPEND - - #turn on glob expansion - set +f + if [[ -z ${_IN_INSTALL_QA_CHECK} ]]; then + #turn off glob expansion + set -f + + # If each var has a value, append it to the global variable E_* to + # be applied after everything is finished. New incremental behavior. + [ "${IUSE+set}" = set ] && E_IUSE+="${E_IUSE:+ }${IUSE}" + [ "${REQUIRED_USE+set}" = set ] && E_REQUIRED_USE+="${E_REQUIRED_USE:+ }${REQUIRED_USE}" + [ "${DEPEND+set}" = set ] && E_DEPEND+="${E_DEPEND:+ }${DEPEND}" + [ "${RDEPEND+set}" = set ] && E_RDEPEND+="${E_RDEPEND:+ }${RDEPEND}" + [ "${PDEPEND+set}" = set ] && E_PDEPEND+="${E_PDEPEND:+ }${PDEPEND}" + [ "${HDEPEND+set}" = set ] && E_HDEPEND+="${E_HDEPEND:+ }${HDEPEND}" + + [ "${B_IUSE+set}" = set ] && IUSE="${B_IUSE}" + [ "${B_IUSE+set}" = set ] || unset IUSE + + [ "${B_REQUIRED_USE+set}" = set ] && REQUIRED_USE="${B_REQUIRED_USE}" + [ "${B_REQUIRED_USE+set}" = set ] || unset REQUIRED_USE + + [ "${B_DEPEND+set}" = set ] && DEPEND="${B_DEPEND}" + [ "${B_DEPEND+set}" = set ] || unset DEPEND + + [ "${B_RDEPEND+set}" = set ] && RDEPEND="${B_RDEPEND}" + [ "${B_RDEPEND+set}" = set ] || unset RDEPEND + + [ "${B_PDEPEND+set}" = set ] && PDEPEND="${B_PDEPEND}" + [ "${B_PDEPEND+set}" = set ] || unset PDEPEND + + [ "${B_HDEPEND+set}" = set ] && HDEPEND="${B_HDEPEND}" + [ "${B_HDEPEND+set}" = set ] || unset HDEPEND + + #turn on glob expansion + set +f + + if [[ -n ${!__export_funcs_var} ]] ; then + for x in ${!__export_funcs_var} ; do + debug-print "EXPORT_FUNCTIONS: $x -> ${ECLASS}_$x" + declare -F "${ECLASS}_$x" >/dev/null || \ + die "EXPORT_FUNCTIONS: ${ECLASS}_$x is not defined" + eval "$x() { ${ECLASS}_$x \"\$@\" ; }" > /dev/null + done + fi + unset $__export_funcs_var - if [[ -n ${!__export_funcs_var} ]] ; then - for x in ${!__export_funcs_var} ; do - debug-print "EXPORT_FUNCTIONS: $x -> ${ECLASS}_$x" - declare -F "${ECLASS}_$x" >/dev/null || \ - die "EXPORT_FUNCTIONS: ${ECLASS}_$x is not defined" - eval "$x() { ${ECLASS}_$x \"\$@\" ; }" > /dev/null - done + has $1 $INHERITED || export INHERITED="$INHERITED $1" fi - unset $__export_funcs_var - - has $1 $INHERITED || export INHERITED="$INHERITED $1" shift done diff --git a/bin/egencache b/bin/egencache index 4ec4108e8..e2f57a58a 100755 --- a/bin/egencache +++ b/bin/egencache @@ -41,8 +41,8 @@ import textwrap import re from os import path as osp -pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") -sys.path.insert(0, pym_path) +if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), ".portage_not_installed")): + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage portage._internal_caller = True from portage import os, _encodings, _unicode_encode, _unicode_decode diff --git a/bin/emaint b/bin/emaint index 00711aa81..784d8613e 100755 --- a/bin/emaint +++ b/bin/emaint @@ -26,8 +26,8 @@ except KeyboardInterrupt: sys.exit(1) from os import path as osp -pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") -sys.path.insert(0, pym_path) +if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), ".portage_not_installed")): + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage portage._internal_caller = True from portage.emaint.main import emaint_main diff --git a/bin/emerge b/bin/emerge index 273c0db04..ae6b1c131 100755 --- a/bin/emerge +++ b/bin/emerge @@ -36,21 +36,27 @@ try: signal.signal(debug_signum, debug_signal) from os import path as osp - pym_path = osp.join(osp.dirname(osp.dirname( - osp.realpath(__file__))), "pym") - sys.path.insert(0, pym_path) + if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), ".portage_not_installed")): + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage portage._internal_caller = True portage._disable_legacy_globals() from _emerge.main import emerge_main if __name__ == "__main__": - from portage.exception import ParseError, PermissionDenied + from portage.exception import IsADirectory, ParseError, \ + PermissionDenied try: retval = emerge_main() except PermissionDenied as e: sys.stderr.write("Permission denied: '%s'\n" % str(e)) sys.exit(e.errno) + except IsADirectory as e: + sys.stderr.write("'%s' is a directory, but should be a file!\n" + "See portage man page for information on " + "which files may be directories.\n" % + str(e)) + sys.exit(e.errno) except ParseError as e: sys.stderr.write("%s\n" % str(e)) sys.exit(1) diff --git a/bin/emerge-webrsync b/bin/emerge-webrsync index 340c35915..dbcc294cf 100755 --- a/bin/emerge-webrsync +++ b/bin/emerge-webrsync @@ -257,12 +257,12 @@ sync_local() { if has metadata-transfer ${FEATURES} ; then __vecho "Updating cache ..." - "${PORTAGE_BIN_PATH}/emerge" --metadata + emerge --metadata fi local post_sync=${PORTAGE_CONFIGROOT}etc/portage/bin/post_sync [ -x "${post_sync}" ] && "${post_sync}" # --quiet suppresses output if there are no relevant news items - has news ${FEATURES} && "${PORTAGE_BIN_PATH}/emerge" --check-news --quiet + has news ${FEATURES} && emerge --check-news --quiet return 0 } diff --git a/bin/env-update b/bin/env-update index aefbf21b6..5b99e66dd 100755 --- a/bin/env-update +++ b/bin/env-update @@ -26,8 +26,8 @@ if len(sys.argv) > 1: usage(1) from os import path as osp -pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") -sys.path.insert(0, pym_path) +if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), ".portage_not_installed")): + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage portage._internal_caller = True diff --git a/bin/fixpackages b/bin/fixpackages index a4e411da5..bc242b794 100755 --- a/bin/fixpackages +++ b/bin/fixpackages @@ -8,8 +8,8 @@ import os import sys from os import path as osp -pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") -sys.path.insert(0, pym_path) +if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), ".portage_not_installed")): + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage portage._internal_caller = True from portage import os diff --git a/bin/glsa-check b/bin/glsa-check index 85d4c4149..78728d43c 100755 --- a/bin/glsa-check +++ b/bin/glsa-check @@ -8,8 +8,8 @@ import sys import codecs from os import path as osp -pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") -sys.path.insert(0, pym_path) +if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), ".portage_not_installed")): + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage portage._internal_caller = True from portage import os diff --git a/bin/install-qa-check.d/05double-D b/bin/install-qa-check.d/05double-D new file mode 100644 index 000000000..1634afd78 --- /dev/null +++ b/bin/install-qa-check.d/05double-D @@ -0,0 +1,17 @@ +# Check for accidential install into ${D}/${D} + +DD_check() { + if [[ -d ${D%/}${D} ]] ; then + local -i INSTALLTOD=0 + while read -r -d $'\0' i ; do + eqawarn "QA Notice: /${i##${D%/}${D}} installed in \${D}/\${D}" + ((INSTALLTOD++)) + done < <(find "${D%/}${D}" -print0) + die "Aborting due to QA concerns: ${INSTALLTOD} files installed in ${D%/}${D}" + fi +} + +DD_check +: # guarantee successful exit + +# vim:ft=sh diff --git a/bin/install-qa-check.d/05prefix b/bin/install-qa-check.d/05prefix new file mode 100644 index 000000000..32561e263 --- /dev/null +++ b/bin/install-qa-check.d/05prefix @@ -0,0 +1,118 @@ +# Prefix specific QA checks + +install_qa_check_prefix() { + [[ ${ED} == ${D} ]] && return + + if [[ -d ${ED}/${D} ]] ; then + find "${ED}/${D}" | \ + while read i ; do + eqawarn "QA Notice: /${i##${ED}/${D}} installed in \${ED}/\${D}" + done + die "Aborting due to QA concerns: files installed in ${ED}/${D}" + fi + + if [[ -d ${ED}/${EPREFIX} ]] ; then + find "${ED}/${EPREFIX}/" | \ + while read i ; do + eqawarn "QA Notice: ${i#${D}} double prefix" + done + die "Aborting due to QA concerns: double prefix files installed" + fi + + if [[ -d ${D} ]] ; then + INSTALLTOD=$(find ${D%/} | egrep -v "^${ED}" | sed -e "s|^${D%/}||" | awk '{if (length($0) <= length("'"${EPREFIX}"'")) { if (substr("'"${EPREFIX}"'", 1, length($0)) != $0) {print $0;} } else if (substr($0, 1, length("'"${EPREFIX}"'")) != "'"${EPREFIX}"'") {print $0;} }') + if [[ -n ${INSTALLTOD} ]] ; then + eqawarn "QA Notice: the following files are outside of the prefix:" + eqawarn "${INSTALLTOD}" + die "Aborting due to QA concerns: there are files installed outside the prefix" + fi + fi + + # all further checks rely on ${ED} existing + [[ -d ${ED} ]] || return + + # check shebangs, bug #282539 + rm -f "${T}"/non-prefix-shebangs-errs + local WHITELIST=" /usr/bin/env " + # this is hell expensive, but how else? + find "${ED}" -executable \! -type d -print0 \ + | xargs -0 grep -H -n -m1 "^#!" \ + | while read f ; + do + local fn=${f%%:*} + local pos=${f#*:} ; pos=${pos%:*} + local line=${f##*:} + # shebang always appears on the first line ;) + [[ ${pos} != 1 ]] && continue + local oldIFS=${IFS} + IFS=$'\r'$'\n'$'\t'" " + line=( ${line#"#!"} ) + IFS=${oldIFS} + [[ ${WHITELIST} == *" ${line[0]} "* ]] && continue + local fp=${fn#${D}} ; fp=/${fp%/*} + # line[0] can be an absolutised path, bug #342929 + local eprefix=$(canonicalize ${EPREFIX}) + local rf=${fn} + # in case we deal with a symlink, make sure we don't replace it + # with a real file (sed -i does that) + if [[ -L ${fn} ]] ; then + rf=$(readlink ${fn}) + [[ ${rf} != /* ]] && rf=${fn%/*}/${rf} + # ignore symlinks pointing to outside prefix + # as seen in sys-devel/native-cctools + [[ $(canonicalize "/${rf#${D}}") != ${eprefix}/* ]] && continue + fi + # does the shebang start with ${EPREFIX}, and does it exist? + if [[ ${line[0]} == ${EPREFIX}/* || ${line[0]} == ${eprefix}/* ]] ; then + if [[ ! -e ${ROOT%/}${line[0]} && ! -e ${D%/}${line[0]} ]] ; then + # hmm, refers explicitly to $EPREFIX, but doesn't exist, + # if it's in PATH that's wrong in any case + if [[ ":${PATH}:" == *":${fp}:"* ]] ; then + echo "${fn#${D}}:${line[0]} (explicit EPREFIX but target not found)" \ + >> "${T}"/non-prefix-shebangs-errs + else + eqawarn "${fn#${D}} has explicit EPREFIX in shebang but target not found (${line[0]})" + fi + fi + continue + fi + # unprefixed shebang, is the script directly in $PATH or an init + # script? + if [[ ":${PATH}:${EPREFIX}/etc/init.d:" == *":${fp}:"* ]] ; then + if [[ -e ${EROOT}${line[0]} || -e ${ED}${line[0]} ]] ; then + # is it unprefixed, but we can just fix it because a + # prefixed variant exists + eqawarn "prefixing shebang of ${fn#${D}}" + # statement is made idempotent on purpose, because + # symlinks may point to the same target, and hence the + # same real file may be sedded multiple times since we + # read the shebangs in one go upfront for performance + # reasons + sed -i -e '1s:^#! \?'"${line[0]}"':#!'"${EPREFIX}"${line[0]}':' "${rf}" + continue + else + # this is definitely wrong: script in $PATH and invalid shebang + echo "${fn#${D}}:${line[0]} (script ${fn##*/} installed in PATH but interpreter ${line[0]} not found)" \ + >> "${T}"/non-prefix-shebangs-errs + fi + else + # unprefixed/invalid shebang, but outside $PATH, this may be + # intended (e.g. config.guess) so remain silent by default + has stricter ${FEATURES} && \ + eqawarn "invalid shebang in ${fn#${D}}: ${line[0]}" + fi + done + if [[ -e "${T}"/non-prefix-shebangs-errs ]] ; then + eqawarn "QA Notice: the following files use invalid (possible non-prefixed) shebangs:" + while read line ; do + eqawarn " ${line}" + done < "${T}"/non-prefix-shebangs-errs + rm -f "${T}"/non-prefix-shebangs-errs + die "Aborting due to QA concerns: invalid shebangs found" + fi +} + +install_qa_check_prefix +: # guarantee successful exit + +# vim:ft=sh diff --git a/bin/install-qa-check.d/10executable-issues b/bin/install-qa-check.d/10executable-issues new file mode 100644 index 000000000..f765749b6 --- /dev/null +++ b/bin/install-qa-check.d/10executable-issues @@ -0,0 +1,140 @@ +# Check for major issues with built executables: insecure RPATHs, +# text relocations, executable stacks + +elf_check() { + if type -P scanelf > /dev/null && ! has binchecks ${RESTRICT}; then + local insecure_rpath=0 tmp_quiet=${PORTAGE_QUIET} + local f x + + # display warnings when using stricter because we die afterwards + if has stricter ${FEATURES} ; then + local PORTAGE_QUIET + fi + + # Make sure we disallow insecure RUNPATH/RPATHs. + # 1) References to PORTAGE_BUILDDIR are banned because it's a + # security risk. We don't want to load files from a + # temporary directory. + # 2) If ROOT != "/", references to ROOT are banned because + # that directory won't exist on the target system. + # 3) Null paths are banned because the loader will search $PWD when + # it finds null paths. + local forbidden_dirs="${PORTAGE_BUILDDIR}" + if [[ -n "${ROOT}" && "${ROOT}" != "/" ]]; then + forbidden_dirs+=" ${ROOT}" + fi + local dir l rpath_files=$(scanelf -F '%F:%r' -qBR "${ED}") + f="" + for dir in ${forbidden_dirs}; do + for l in $(echo "${rpath_files}" | grep -E ":${dir}|::|: "); do + f+=" ${l%%:*}\n" + if ! has stricter ${FEATURES}; then + __vecho "Auto fixing rpaths for ${l%%:*}" + TMPDIR="${dir}" scanelf -BXr "${l%%:*}" -o /dev/null + fi + done + done + + # Reject set*id binaries with $ORIGIN in RPATH #260331 + x=$( + find "${ED}" -type f \( -perm -u+s -o -perm -g+s \) -print0 | \ + xargs -0 scanelf -qyRF '%r %p' | grep '$ORIGIN' + ) + + # Print QA notice. + if [[ -n ${f}${x} ]] ; then + __vecho -ne '\n' + eqawarn "QA Notice: The following files contain insecure RUNPATHs" + eqawarn " Please file a bug about this at http://bugs.gentoo.org/" + eqawarn " with the maintaining herd of the package." + eqawarn "${f}${f:+${x:+\n}}${x}" + __vecho -ne '\n' + if [[ -n ${x} ]] || has stricter ${FEATURES} ; then + insecure_rpath=1 + fi + fi + + # TEXTRELs are baaaaaaaad + # Allow devs to mark things as ignorable ... e.g. things that are + # binary-only and upstream isn't cooperating (nvidia-glx) ... we + # allow ebuild authors to set QA_TEXTRELS_arch and QA_TEXTRELS ... + # the former overrides the latter ... regexes allowed ! :) + local qa_var="QA_TEXTRELS_${ARCH/-/_}" + [[ -n ${!qa_var} ]] && QA_TEXTRELS=${!qa_var} + [[ -n ${QA_STRICT_TEXTRELS} ]] && QA_TEXTRELS="" + export QA_TEXTRELS="${QA_TEXTRELS} lib*/modules/*.ko" + f=$(scanelf -qyRF '%t %p' "${ED}" | grep -v 'usr/lib/debug/') + if [[ -n ${f} ]] ; then + scanelf -qyRAF '%T %p' "${PORTAGE_BUILDDIR}"/ &> "${T}"/scanelf-textrel.log + __vecho -ne '\n' + eqawarn "QA Notice: The following files contain runtime text relocations" + eqawarn " Text relocations force the dynamic linker to perform extra" + eqawarn " work at startup, waste system resources, and may pose a security" + eqawarn " risk. On some architectures, the code may not even function" + eqawarn " properly, if at all." + eqawarn " For more information, see http://hardened.gentoo.org/pic-fix-guide.xml" + eqawarn " Please include the following list of files in your report:" + eqawarn "${f}" + __vecho -ne '\n' + die_msg="${die_msg} textrels," + sleep 1 + fi + + # Also, executable stacks only matter on linux (and just glibc atm ...) + f="" + case ${CTARGET:-${CHOST}} in + *-linux-gnu*) + # Check for files with executable stacks, but only on arches which + # are supported at the moment. Keep this list in sync with + # http://www.gentoo.org/proj/en/hardened/gnu-stack.xml (Arch Status) + case ${CTARGET:-${CHOST}} in + arm*|i?86*|ia64*|m68k*|s390*|sh*|x86_64*) + # Allow devs to mark things as ignorable ... e.g. things + # that are binary-only and upstream isn't cooperating ... + # we allow ebuild authors to set QA_EXECSTACK_arch and + # QA_EXECSTACK ... the former overrides the latter ... + # regexes allowed ! :) + + qa_var="QA_EXECSTACK_${ARCH/-/_}" + [[ -n ${!qa_var} ]] && QA_EXECSTACK=${!qa_var} + [[ -n ${QA_STRICT_EXECSTACK} ]] && QA_EXECSTACK="" + qa_var="QA_WX_LOAD_${ARCH/-/_}" + [[ -n ${!qa_var} ]] && QA_WX_LOAD=${!qa_var} + [[ -n ${QA_STRICT_WX_LOAD} ]] && QA_WX_LOAD="" + export QA_EXECSTACK="${QA_EXECSTACK} lib*/modules/*.ko" + export QA_WX_LOAD="${QA_WX_LOAD} lib*/modules/*.ko" + f=$(scanelf -qyRAF '%e %p' "${ED}" | grep -v 'usr/lib/debug/') + ;; + esac + ;; + esac + if [[ -n ${f} ]] ; then + # One more pass to help devs track down the source + scanelf -qyRAF '%e %p' "${PORTAGE_BUILDDIR}"/ &> "${T}"/scanelf-execstack.log + __vecho -ne '\n' + eqawarn "QA Notice: The following files contain writable and executable sections" + eqawarn " Files with such sections will not work properly (or at all!) on some" + eqawarn " architectures/operating systems. A bug should be filed at" + eqawarn " http://bugs.gentoo.org/ to make sure the issue is fixed." + eqawarn " For more information, see http://hardened.gentoo.org/gnu-stack.xml" + eqawarn " Please include the following list of files in your report:" + eqawarn " Note: Bugs should be filed for the respective maintainers" + eqawarn " of the package in question and not hardened@g.o." + eqawarn "${f}" + __vecho -ne '\n' + die_msg="${die_msg} execstacks" + sleep 1 + fi + + if [[ ${insecure_rpath} -eq 1 ]] ; then + die "Aborting due to serious QA concerns with RUNPATH/RPATH" + elif [[ -n ${die_msg} ]] && has stricter ${FEATURES} ; then + die "Aborting due to QA concerns: ${die_msg}" + fi + fi +} + +elf_check +: # guarantee successful exit + +# vim:ft=sh diff --git a/bin/install-qa-check.d/10ignored-flags b/bin/install-qa-check.d/10ignored-flags new file mode 100644 index 000000000..7aa9eb695 --- /dev/null +++ b/bin/install-qa-check.d/10ignored-flags @@ -0,0 +1,99 @@ +# QA checks for ignored *FLAGS. + +ignored_flag_check() { + type -P scanelf > /dev/null || return + has binchecks ${RESTRICT} && return + + local qa_var="QA_FLAGS_IGNORED_${ARCH/-/_}" + eval "[[ -n \${!qa_var} ]] && QA_FLAGS_IGNORED=(\"\${${qa_var}[@]}\")" + if [[ ${#QA_FLAGS_IGNORED[@]} -eq 1 ]] ; then + local shopts=$- + set -o noglob + QA_FLAGS_IGNORED=(${QA_FLAGS_IGNORED}) + set +o noglob + set -${shopts} + fi + + local f x + + # Check for files built without respecting *FLAGS. Note that + # -frecord-gcc-switches must be in all *FLAGS variables, in + # order to avoid false positive results here. + # NOTE: This check must execute before prepall/prepstrip, since + # prepstrip strips the .GCC.command.line sections. + if [[ "${CFLAGS}" == *-frecord-gcc-switches* ]] && \ + [[ "${CXXFLAGS}" == *-frecord-gcc-switches* ]] && \ + [[ "${FFLAGS}" == *-frecord-gcc-switches* ]] && \ + [[ "${FCFLAGS}" == *-frecord-gcc-switches* ]] ; then + rm -f "${T}"/scanelf-ignored-CFLAGS.log + for x in $(scanelf -qyRF '#k%p' -k '!.GCC.command.line' "${ED}") ; do + # Separate out file types that are known to support + # .GCC.command.line sections, using the `file` command + # similar to how prepstrip uses it. + f=$(file "${x}") || continue + [[ -z ${f} ]] && continue + if [[ ${f} == *"SB executable"* || + ${f} == *"SB shared object"* ]] ; then + echo "${x}" >> "${T}"/scanelf-ignored-CFLAGS.log + fi + done + + if [[ -f "${T}"/scanelf-ignored-CFLAGS.log ]] ; then + + if [ "${QA_STRICT_FLAGS_IGNORED-unset}" = unset ] ; then + for x in "${QA_FLAGS_IGNORED[@]}" ; do + sed -e "s#^${x#/}\$##" -i "${T}"/scanelf-ignored-CFLAGS.log + done + fi + # Filter anything under /usr/lib/debug/ in order to avoid + # duplicate warnings for splitdebug files. + sed -e "s#^usr/lib/debug/.*##" -e "/^\$/d" -e "s#^#/#" \ + -i "${T}"/scanelf-ignored-CFLAGS.log + f=$(<"${T}"/scanelf-ignored-CFLAGS.log) + if [[ -n ${f} ]] ; then + __vecho -ne '\n' + eqawarn "${BAD}QA Notice: Files built without respecting CFLAGS have been detected${NORMAL}" + eqawarn " Please include the following list of files in your report:" + eqawarn "${f}" + __vecho -ne '\n' + sleep 1 + else + rm -f "${T}"/scanelf-ignored-CFLAGS.log + fi + fi + fi + + # Check for files built without respecting LDFLAGS + if [[ "${LDFLAGS}" == *,--hash-style=gnu* ]] && \ + ! has binchecks ${RESTRICT} ; then + f=$(scanelf -qyRF '#k%p' -k .hash "${ED}") + if [[ -n ${f} ]] ; then + echo "${f}" > "${T}"/scanelf-ignored-LDFLAGS.log + if [ "${QA_STRICT_FLAGS_IGNORED-unset}" = unset ] ; then + for x in "${QA_FLAGS_IGNORED[@]}" ; do + sed -e "s#^${x#/}\$##" -i "${T}"/scanelf-ignored-LDFLAGS.log + done + fi + # Filter anything under /usr/lib/debug/ in order to avoid + # duplicate warnings for splitdebug files. + sed -e "s#^usr/lib/debug/.*##" -e "/^\$/d" -e "s#^#/#" \ + -i "${T}"/scanelf-ignored-LDFLAGS.log + f=$(<"${T}"/scanelf-ignored-LDFLAGS.log) + if [[ -n ${f} ]] ; then + __vecho -ne '\n' + eqawarn "${BAD}QA Notice: Files built without respecting LDFLAGS have been detected${NORMAL}" + eqawarn " Please include the following list of files in your report:" + eqawarn "${f}" + __vecho -ne '\n' + sleep 1 + else + rm -f "${T}"/scanelf-ignored-LDFLAGS.log + fi + fi + fi +} + +ignored_flag_check +: # guarantee successful exit + +# vim:ft=sh diff --git a/bin/install-qa-check.d/20deprecated-directories b/bin/install-qa-check.d/20deprecated-directories new file mode 100644 index 000000000..fb82bfe7a --- /dev/null +++ b/bin/install-qa-check.d/20deprecated-directories @@ -0,0 +1,18 @@ +# Check for deprecated directories + +deprecated_dir_check() { + local x f= + for x in etc/app-defaults usr/man usr/info usr/X11R6 usr/doc usr/locale ; do + [[ -d ${ED}/$x ]] && f+=" $x\n" + done + if [[ -n $f ]] ; then + eqawarn "QA Notice: This ebuild installs into the following deprecated directories:" + eqawarn + eqawarn "$f" + fi +} + +deprecated_dir_check +: # guarantee successful exit + +# vim:ft=sh diff --git a/bin/install-qa-check.d/20runtime-directories b/bin/install-qa-check.d/20runtime-directories new file mode 100644 index 000000000..2e21d6d04 --- /dev/null +++ b/bin/install-qa-check.d/20runtime-directories @@ -0,0 +1,26 @@ +# Check for directories that need to be created at runtime + +runtime_dir_check() { + # It's ok create these directories, but not to install into them. #493154 + # TODO: We should add var/lib to this list. + local x f= + for x in var/cache var/lock var/run run ; do + if [[ ! -L ${ED}/${x} && -d ${ED}/${x} ]] ; then + if [[ -z $(find "${ED}/${x}" -prune -empty) ]] ; then + f+=$(cd "${ED}"; find "${x}" -printf ' %p\n') + fi + fi + done + if [[ -n ${f} ]] ; then + eqawarn "QA Notice: This ebuild installs into paths that should be created at runtime." + eqawarn " To fix, simply do not install into these directories. Instead, your package" + eqawarn " should create dirs on the fly at runtime as needed via init scripts/etc..." + eqawarn + eqawarn "${f}" + fi +} + +runtime_dir_check +: # guarantee successful exit + +# vim:ft=sh diff --git a/bin/install-qa-check.d/60bash-completion b/bin/install-qa-check.d/60bash-completion new file mode 100644 index 000000000..c1547619a --- /dev/null +++ b/bin/install-qa-check.d/60bash-completion @@ -0,0 +1,130 @@ +# QA checks for bash-completion files + +bashcomp_check() { + # Check for correct bash-completion install path. + local syscompdir=$(pkg-config --variable=completionsdir bash-completion 2>/dev/null) + : ${syscompdir:=${EPREFIX}/usr/share/bash-completion/completions} + + local instcompdir + if [[ -d ${ED}/usr/share/bash-completion/completions ]]; then + instcompdir=${ED}/usr/share/bash-completion/completions + elif [[ -d ${ED}/usr/share/bash-completion ]]; then + if [[ ${syscompdir} != ${EPREFIX}/usr/share/bash-completion ]]; then + eqawarn "Bash completions were installed in legacy location. Please update" + eqawarn "the ebuild to get the install paths using bash-completion-r1.eclass." + eqawarn + fi + + instcompdir=${ED}/usr/share/bash-completion + fi + + # Do a few QA tests on bash completions. + if [[ -n ${instcompdir} && -f ${EROOT}/usr/share/bash-completion/bash_completion ]]; then + _get_completions() { + # source the file + source "${1}" &>/dev/null + + [[ ${USED_HAVE} == yes ]] && echo '__HAVE_USED__' + + # print the completed commands + while read -a args; do + [[ ${args[0]} == complete ]] || continue + # command always comes last, one per line + echo "${args[$(( ${#args[@]} - 1))]}" + done < <(complete -p) + } + + # load the global helpers + source "${EROOT}"/usr/share/bash-completion/bash_completion + + # clean up predefined completions + complete -r + + # force all completions on + _have() { + return 0 + } + + local USED_HAVE=no + # add a replacement for have() + have() { + USED_HAVE=yes + + unset -v have + _have ${1} && have=yes + } + + local f c completions + local all_compls=() + local all_files=() + local qa_warnings=() + + for f in "${instcompdir}"/*; do + # ignore directories and other non-files + [[ ! -f ${f} ]] && continue + + # skip the common code file + # (in case we're run in /usr/share/bash-completion) + [[ ${f##*/} == bash_completion ]] && continue + + completions=( $(_get_completions "${f}") ) + + if [[ ${completions[0]} == __HAVE_USED__ ]]; then + qa_warnings+=( + "${f##*/}: 'have' command is deprecated and must not be used." + ) + unset 'completions[0]' + fi + + if [[ -z ${completions[@]} ]]; then + qa_warnings+=( + "${f##*/}: does not define any completions (failed to source?)." + ) + continue + fi + + for c in "${completions[@]}"; do + if [[ ${c} == /* ]]; then + qa_warnings+=( + "${f##*/}: absolute paths can not be used for completions (on '${c}')." + ) + else + all_compls+=( "${c}" ) + fi + done + + if ! has "${f##*/}" "${all_compls[@]}"; then + qa_warnings+=( + "${f##*/}: incorrect name, no completions for '${f##*/}' command defined." + ) + fi + + all_files+=( "${f##*/}" ) + done + + for c in "${all_compls[@]}"; do + if ! has "${c}" "${all_files[@]}"; then + qa_warnings+=( + "${c}: missing alias (symlink) for completed command." + ) + fi + done + + if [[ -n ${qa_warnings[@]} ]]; then + eqawarn "Problems with installed bash completions were found:" + eqawarn + for c in "${qa_warnings[@]}"; do + eqawarn " ${c}" + done + eqawarn + eqawarn "For more details on installing bash-completions, please see:" + eqawarn "https://wiki.gentoo.org/wiki/Bash/Installing_completion_files" + eqawarn + fi + fi +} + +bashcomp_check +: # guarantee successful exit + +# vim:ft=sh diff --git a/bin/install-qa-check.d/60openrc b/bin/install-qa-check.d/60openrc new file mode 100644 index 000000000..9b7fc6d99 --- /dev/null +++ b/bin/install-qa-check.d/60openrc @@ -0,0 +1,41 @@ +# QA checks for OpenRC init.d files. + +openrc_check() { + # Sanity check syntax errors in init.d scripts + local d i + for d in /etc/conf.d /etc/init.d ; do + [[ -d ${ED}/${d} ]] || continue + for i in "${ED}"/${d}/* ; do + [[ -L ${i} ]] && continue + # if empty conf.d/init.d dir exists (baselayout), then i will be "/etc/conf.d/*" and not exist + [[ ! -e ${i} ]] && continue + if [[ ${d} == /etc/init.d && ${i} != *.sh ]] ; then + # skip non-shell-script for bug #451386 + [[ $(head -n1 "${i}") =~ ^#!.*[[:space:]/](runscript|sh)$ ]] || continue + fi + bash -n "${i}" || die "The init.d file has syntax errors: ${i}" + done + done + + local checkbashisms=$(type -P checkbashisms) + if [[ -n ${checkbashisms} ]] ; then + for d in /etc/init.d ; do + [[ -d ${ED}${d} ]] || continue + for i in "${ED}${d}"/* ; do + [[ -e ${i} ]] || continue + [[ -L ${i} ]] && continue + f=$("${checkbashisms}" -f "${i}" 2>&1) + [[ $? != 0 && -n ${f} ]] || continue + eqawarn "QA Notice: shell script appears to use non-POSIX feature(s):" + while read -r ; + do eqawarn " ${REPLY}" + done <<< "${f//${ED}}" + done + done + fi +} + +openrc_check +: # guarantee successful exit + +# vim:ft=sh diff --git a/bin/install-qa-check.d/60pkgconfig b/bin/install-qa-check.d/60pkgconfig new file mode 100644 index 000000000..1b34c04db --- /dev/null +++ b/bin/install-qa-check.d/60pkgconfig @@ -0,0 +1,15 @@ +# Check for pkg-config file issues + +pkgconfig_check() { + # Look for leaking LDFLAGS into pkg-config files + local f=$(egrep -sH '^Libs.*-Wl,(-O[012]|--hash-style)' "${ED}"/usr/*/pkgconfig/*.pc) + if [[ -n ${f} ]] ; then + eqawarn "QA Notice: pkg-config files with wrong LDFLAGS detected:" + eqawarn "${f//${D}}" + fi +} + +pkgconfig_check +: # guarantee successful exit + +# vim:ft=sh diff --git a/bin/install-qa-check.d/60pngfix b/bin/install-qa-check.d/60pngfix new file mode 100644 index 000000000..8d53040b6 --- /dev/null +++ b/bin/install-qa-check.d/60pngfix @@ -0,0 +1,35 @@ +# Check for issues with PNG files + +pngfix_check() { + local pngfix=$(type -P pngfix) + if [[ -n ${pngfix} ]] ; then + local pngout=() + local next + + while read -r -a pngout ; do + local error="" + + case "${pngout[1]}" in + CHK) + error='invalid checksum' + ;; + TFB) + error='broken IDAT window length' + ;; + esac + + if [[ -n ${error} ]] ; then + if [[ -z ${next} ]] ; then + eqawarn "QA Notice: broken .png files found:" + next=1 + fi + eqawarn " ${pngout[@]:7}: ${error}" + fi + done < <(find "${ED}" -type f -name '*.png' -exec "${pngfix}" {} +) + fi +} + +pngfix_check +: # guarantee successful exit + +# vim:ft=sh diff --git a/bin/install-qa-check.d/60systemd b/bin/install-qa-check.d/60systemd new file mode 100644 index 000000000..f134a3068 --- /dev/null +++ b/bin/install-qa-check.d/60systemd @@ -0,0 +1,25 @@ +# QA checks for systemd units. + +systemd_check() { + local systemddir f + + # Common mistakes in systemd service files. + if type -P pkg-config >/dev/null && pkg-config --exists systemd; then + systemddir=$(pkg-config --variable=systemdsystemunitdir systemd) + else + systemddir=/usr/lib/systemd/system + fi + if [[ -d ${ED%/}${systemddir} ]]; then + f=$(grep -sH '^EnvironmentFile.*=.*/etc/conf\.d' "${ED%/}${systemddir}"/*.service) + if [[ -n ${f} ]] ; then + eqawarn "QA Notice: systemd units using /etc/conf.d detected:" + eqawarn "${f//${D}}" + eqawarn "See: https://wiki.gentoo.org/wiki/Project:Systemd/conf.d_files" + fi + fi +} + +systemd_check +: # guarantee successful exit + +# vim:ft=sh diff --git a/bin/install-qa-check.d/60udev b/bin/install-qa-check.d/60udev new file mode 100644 index 000000000..4327d069d --- /dev/null +++ b/bin/install-qa-check.d/60udev @@ -0,0 +1,21 @@ +# Check udev rule installs + +udev_check() { + set +f + local x f= + for x in "${ED}etc/udev/rules.d/"* "${ED}lib"*"/udev/rules.d/"* ; do + [[ -e ${x} ]] || continue + [[ ${x} == ${ED}lib/udev/rules.d/* ]] && continue + f+=" ${x#${ED}}\n" + done + if [[ -n $f ]] ; then + eqawarn "QA Notice: udev rules should be installed in /lib/udev/rules.d:" + eqawarn + eqawarn "$f" + fi +} + +udev_check +: # guarantee successful exit + +# vim:ft=sh diff --git a/bin/install-qa-check.d/80libraries b/bin/install-qa-check.d/80libraries new file mode 100644 index 000000000..c83f27800 --- /dev/null +++ b/bin/install-qa-check.d/80libraries @@ -0,0 +1,167 @@ +# Check for issues with installed libraries + +lib_check() { + local f x i j + + if type -P scanelf > /dev/null && ! has binchecks ${RESTRICT}; then + # Check for shared libraries lacking SONAMEs + local qa_var="QA_SONAME_${ARCH/-/_}" + eval "[[ -n \${!qa_var} ]] && QA_SONAME=(\"\${${qa_var}[@]}\")" + f=$(scanelf -ByF '%S %p' "${ED}"{,usr/}lib*/lib*.so* | awk '$2 == "" { print }' | sed -e "s:^[[:space:]]${ED}:/:") + if [[ -n ${f} ]] ; then + echo "${f}" > "${T}"/scanelf-missing-SONAME.log + if [[ "${QA_STRICT_SONAME-unset}" == unset ]] ; then + if [[ ${#QA_SONAME[@]} -gt 1 ]] ; then + for x in "${QA_SONAME[@]}" ; do + sed -e "s#^/${x#/}\$##" -i "${T}"/scanelf-missing-SONAME.log + done + else + local shopts=$- + set -o noglob + for x in ${QA_SONAME} ; do + sed -e "s#^/${x#/}\$##" -i "${T}"/scanelf-missing-SONAME.log + done + set +o noglob + set -${shopts} + fi + fi + sed -e "/^\$/d" -i "${T}"/scanelf-missing-SONAME.log + f=$(<"${T}"/scanelf-missing-SONAME.log) + if [[ -n ${f} ]] ; then + __vecho -ne '\n' + eqawarn "QA Notice: The following shared libraries lack a SONAME" + eqawarn "${f}" + __vecho -ne '\n' + sleep 1 + else + rm -f "${T}"/scanelf-missing-SONAME.log + fi + fi + + # Check for shared libraries lacking NEEDED entries + qa_var="QA_DT_NEEDED_${ARCH/-/_}" + eval "[[ -n \${!qa_var} ]] && QA_DT_NEEDED=(\"\${${qa_var}[@]}\")" + f=$(scanelf -ByF '%n %p' "${ED}"{,usr/}lib*/lib*.so* | awk '$2 == "" { print }' | sed -e "s:^[[:space:]]${ED}:/:") + if [[ -n ${f} ]] ; then + echo "${f}" > "${T}"/scanelf-missing-NEEDED.log + if [[ "${QA_STRICT_DT_NEEDED-unset}" == unset ]] ; then + if [[ ${#QA_DT_NEEDED[@]} -gt 1 ]] ; then + for x in "${QA_DT_NEEDED[@]}" ; do + sed -e "s#^/${x#/}\$##" -i "${T}"/scanelf-missing-NEEDED.log + done + else + local shopts=$- + set -o noglob + for x in ${QA_DT_NEEDED} ; do + sed -e "s#^/${x#/}\$##" -i "${T}"/scanelf-missing-NEEDED.log + done + set +o noglob + set -${shopts} + fi + fi + sed -e "/^\$/d" -i "${T}"/scanelf-missing-NEEDED.log + f=$(<"${T}"/scanelf-missing-NEEDED.log) + if [[ -n ${f} ]] ; then + __vecho -ne '\n' + eqawarn "QA Notice: The following shared libraries lack NEEDED entries" + eqawarn "${f}" + __vecho -ne '\n' + sleep 1 + else + rm -f "${T}"/scanelf-missing-NEEDED.log + fi + fi + fi + + # this should help to ensure that all (most?) shared libraries are executable + # and that all libtool scripts / static libraries are not executable + for i in "${ED}"opt/*/lib* \ + "${ED}"lib* \ + "${ED}"usr/lib* ; do + [[ ! -d ${i} ]] && continue + + for j in "${i}"/*.so.* "${i}"/*.so ; do + [[ ! -e ${j} ]] && continue + [[ -L ${j} ]] && continue + [[ -x ${j} ]] && continue + __vecho "making executable: ${j#${ED}}" + chmod +x "${j}" + done + + for j in "${i}"/*.a "${i}"/*.la ; do + [[ ! -e ${j} ]] && continue + [[ -L ${j} ]] && continue + [[ ! -x ${j} ]] && continue + __vecho "removing executable bit: ${j#${ED}}" + chmod -x "${j}" + done + + for j in "${i}"/*.{a,dll,dylib,sl,so}.* "${i}"/*.{a,dll,dylib,sl,so} ; do + [[ ! -e ${j} ]] && continue + [[ ! -L ${j} ]] && continue + linkdest=$(readlink "${j}") + if [[ ${linkdest} == /* ]] ; then + __vecho -ne '\n' + eqawarn "QA Notice: Found an absolute symlink in a library directory:" + eqawarn " ${j#${D}} -> ${linkdest}" + eqawarn " It should be a relative symlink if in the same directory" + eqawarn " or a linker script if it crosses the /usr boundary." + fi + done + done + + # When installing static libraries into /usr/lib and shared libraries into + # /lib, we have to make sure we have a linker script in /usr/lib along side + # the static library, or gcc will utilize the static lib when linking :(. + # http://bugs.gentoo.org/4411 + local abort="no" + local a s + for a in "${ED}"usr/lib*/*.a ; do + # PREFIX LOCAL: support MachO objects + [[ ${CHOST} == *-darwin* ]] \ + && s=${a%.a}.dylib \ + || s=${a%.a}.so + # END PREFIX LOCAL + if [[ ! -e ${s} ]] ; then + s=${s%usr/*}${s##*/usr/} + if [[ -e ${s} ]] ; then + __vecho -ne '\n' + eqawarn "QA Notice: Missing gen_usr_ldscript for ${s##*/}" + abort="yes" + fi + fi + done + [[ ${abort} == "yes" ]] && die "add those ldscripts" + + # Make sure people don't store libtool files or static libs in /lib + # PREFIX LOCAL: on AIX, "dynamic libs" have extension .a, so don't + # get false positives + [[ ${CHOST} == *-aix* ]] \ + && f=$(ls "${ED}"lib*/*.la 2>/dev/null || true) \ + || f=$(ls "${ED}"lib*/*.{a,la} 2>/dev/null) + # END PREFIX LOCAL + if [[ -n ${f} ]] ; then + __vecho -ne '\n' + eqawarn "QA Notice: Excessive files found in the / partition" + eqawarn "${f}" + __vecho -ne '\n' + die "static archives (*.a) and libtool library files (*.la) belong in /usr/lib*, not /lib*" + fi + + # Verify that the libtool files don't contain bogus $D entries. + local abort=no gentoo_bug=no always_overflow=no + for a in "${ED}"usr/lib*/*.la ; do + s=${a##*/} + if grep -qs "${ED}" "${a}" ; then + __vecho -ne '\n' + eqawarn "QA Notice: ${s} appears to contain PORTAGE_TMPDIR paths" + abort="yes" + fi + done + [[ ${abort} == "yes" ]] && die "soiled libtool library files found" +} + +lib_check +: # guarantee successful exit + +# vim:ft=sh diff --git a/bin/install-qa-check.d/80multilib-strict b/bin/install-qa-check.d/80multilib-strict new file mode 100644 index 000000000..436932e80 --- /dev/null +++ b/bin/install-qa-check.d/80multilib-strict @@ -0,0 +1,50 @@ +# Strict multilib directory checks +multilib_strict_check() { + if has multilib-strict ${FEATURES} && \ + [[ -x ${EPREFIX}/usr/bin/file && -x ${EPREFIX}/usr/bin/find ]] && \ + [[ -n ${MULTILIB_STRICT_DIRS} && -n ${MULTILIB_STRICT_DENY} ]] + then + rm -f "${T}/multilib-strict.log" + local abort=no dir file + MULTILIB_STRICT_EXEMPT=$(echo ${MULTILIB_STRICT_EXEMPT} | sed -e 's:\([(|)]\):\\\1:g') + for dir in ${MULTILIB_STRICT_DIRS} ; do + [[ -d ${ED}/${dir} ]] || continue + for file in $(find ${ED}/${dir} -type f | grep -v "^${ED}/${dir}/${MULTILIB_STRICT_EXEMPT}"); do + if file ${file} | egrep -q "${MULTILIB_STRICT_DENY}" ; then + echo "${file#${ED}//}" >> "${T}/multilib-strict.log" + fi + done + done + + if [[ -s ${T}/multilib-strict.log ]] ; then + if [[ ${#QA_MULTILIB_PATHS[@]} -eq 1 ]] ; then + local shopts=$- + set -o noglob + QA_MULTILIB_PATHS=(${QA_MULTILIB_PATHS}) + set +o noglob + set -${shopts} + fi + if [ "${QA_STRICT_MULTILIB_PATHS-unset}" = unset ] ; then + local x + for x in "${QA_MULTILIB_PATHS[@]}" ; do + sed -e "s#^${x#/}\$##" -i "${T}/multilib-strict.log" + done + sed -e "/^\$/d" -i "${T}/multilib-strict.log" + fi + if [[ -s ${T}/multilib-strict.log ]] ; then + abort=yes + echo "Files matching a file type that is not allowed:" + while read -r ; do + echo " ${REPLY}" + done < "${T}/multilib-strict.log" + fi + fi + + [[ ${abort} == yes ]] && die "multilib-strict check failed!" + fi +} + +multilib_strict_check +: # guarantee successful exit + +# vim:ft=sh diff --git a/bin/install-qa-check.d/90gcc-warnings b/bin/install-qa-check.d/90gcc-warnings new file mode 100644 index 000000000..9703c39de --- /dev/null +++ b/bin/install-qa-check.d/90gcc-warnings @@ -0,0 +1,168 @@ +# Check for important gcc warning + +gcc_warn_check() { + local f + + # Evaluate misc gcc warnings + if [[ -n ${PORTAGE_LOG_FILE} && -r ${PORTAGE_LOG_FILE} ]] ; then + # In debug mode, this variable definition and corresponding grep calls + # will produce false positives if they're shown in the trace. + local reset_debug=0 + if [[ ${-/x/} != $- ]] ; then + set +x + reset_debug=1 + fi + local m msgs=( + # only will and does, no might :) + 'warning: .*will.*\[-Wstrict-aliasing\]' + 'warning: .*does.*\[-Wstrict-aliasing\]' + # implicit declaration of function ‘...’ + 'warning: .*\[-Wimplicit-function-declaration\]' + # with -Wall, goes in pair with -Wimplicit-function-declaration + # but without -Wall, we need to assert for it alone + 'warning: .*incompatible implicit declaration of built-in function' + # 'is used uninitialized in this function' and some more + 'warning: .*\[-Wuninitialized\]' + # comparisons like ‘X<=Y<=Z’ do not have their mathematical meaning + 'warning: .*mathematical meaning*\[-Wparentheses\]' + # null argument where non-null required + 'warning: .*\[-Wnonnull\]' + # array subscript is above/below/outside array bounds + 'warning: .*\[-Warray-bounds\]' + # attempt to free a non-heap object + 'warning: .*\[-Wfree-nonheap-object\]' + # those three do not have matching -W flags, it seems + 'warning: .*will always overflow destination buffer' + 'warning: .*assuming pointer wraparound does not occur' + 'warning: .*escape sequence out of range' + # left/right-hand operand of comma expression has no effect + 'warning: .*comma.*\[-Wunused-value\]' + # converting to non-pointer type ... from NULL and likes + 'warning: .*\[-Wconversion-null\]' + # NULL used in arithmetic + 'warning: .*NULL.*\[-Wpointer-arith\]' + # pointer to a function used in arithmetic and likes + 'warning: .*function.*\[-Wpointer-arith\]' + # the address of ... will never be NULL and likes + # (uses of function refs & string constants in conditionals) + 'warning: .*\[-Waddress\]' + # outdated? + 'warning: .*too few arguments for format' + # format ... expects a matching ... argument + # (iow, too few arguments for format in new wording :)) + 'warning: .*matching.*\[-Wformat=\]' + # function returns address of local variable + 'warning: .*\[-Wreturn-local-addr\]' + # argument to sizeof ... is the same expression as the source + 'warning: .*\[-Wsizeof-pointer-memaccess\]' + # iteration invokes undefined behavior + 'warning: .*\[-Waggressive-loop-optimizations\]' + + # this may be valid code :/ + #': warning: multi-character character constant' + # need to check these two ... + #': warning: assuming signed overflow does not occur when' + #': warning: comparison with string literal results in unspecified behav' + # yacc/lex likes to trigger this one + #': warning: extra tokens at end of .* directive' + # only gcc itself triggers this ? + #': warning: .*noreturn.* function does return' + # these throw false positives when 0 is used instead of NULL + #': warning: missing sentinel in function call' + #': warning: not enough variable arguments to fit a sentinel' + ) + + # join all messages into one grep-expression + local joined_msgs + printf -v joined_msgs '%s|' "${msgs[@]}" + joined_msgs=${joined_msgs%|} + + local abort="no" + local grep_cmd=grep + [[ $PORTAGE_LOG_FILE = *.gz ]] && grep_cmd=zgrep + + # force C locale to work around slow unicode locales #160234 + f=$(LC_CTYPE=C LC_COLLATE=C "${grep_cmd}" -E "${joined_msgs}" "${PORTAGE_LOG_FILE}") + if [[ -n ${f} ]] ; then + abort="yes" + # for now, don't make this fatal (see bug #337031) + #if [[ ${f} == *'will always overflow destination buffer'* ]]; then + # always_overflow=yes + #fi + if [[ $always_overflow = yes ]] ; then + eerror + eerror "QA Notice: Package triggers severe warnings which indicate that it" + eerror " may exhibit random runtime failures." + eerror + eerror "${f}" + eerror + eerror " Please file a bug about this at http://bugs.gentoo.org/" + eerror " with the maintaining herd of the package." + eerror + else + __vecho -ne '\n' + eqawarn "QA Notice: Package triggers severe warnings which indicate that it" + eqawarn " may exhibit random runtime failures." + eqawarn "${f}" + __vecho -ne '\n' + fi + fi + + local cat_cmd=cat + [[ $PORTAGE_LOG_FILE = *.gz ]] && cat_cmd=zcat + [[ $reset_debug = 1 ]] && set -x + # Use safe cwd, avoiding unsafe import for bug #469338. + f=$(cd "${PORTAGE_PYM_PATH}" ; $cat_cmd "${PORTAGE_LOG_FILE}" | \ + "${PORTAGE_PYTHON:-/usr/bin/python}" "$PORTAGE_BIN_PATH"/check-implicit-pointer-usage.py || die "check-implicit-pointer-usage.py failed") + if [[ -n ${f} ]] ; then + + # In the future this will be a forced "die". In preparation, + # increase the log level from "qa" to "eerror" so that people + # are aware this is a problem that must be fixed asap. + + # just warn on 32bit hosts but bail on 64bit hosts + case ${CHOST} in + alpha*|hppa64*|ia64*|powerpc64*|mips64*|sparc64*|sparcv9*|x86_64*) gentoo_bug=yes ;; + esac + + abort=yes + + if [[ $gentoo_bug = yes ]] ; then + eerror + eerror "QA Notice: Package triggers severe warnings which indicate that it" + eerror " will almost certainly crash on 64bit architectures." + eerror + eerror "${f}" + eerror + eerror " Please file a bug about this at http://bugs.gentoo.org/" + eerror " with the maintaining herd of the package." + eerror + else + __vecho -ne '\n' + eqawarn "QA Notice: Package triggers severe warnings which indicate that it" + eqawarn " will almost certainly crash on 64bit architectures." + eqawarn "${f}" + __vecho -ne '\n' + fi + + fi + if [[ ${abort} == "yes" ]] ; then + if [[ $gentoo_bug = yes || $always_overflow = yes ]] ; then + die "install aborted due to severe warnings shown above" + else + echo "Please do not file a Gentoo bug and instead" \ + "report the above QA issues directly to the upstream" \ + "developers of this software." | fmt -w 70 | \ + while read -r line ; do eqawarn "${line}" ; done + eqawarn "Homepage: ${HOMEPAGE}" + has stricter ${FEATURES} && \ + die "install aborted due to severe warnings shown above" + fi + fi + fi +} + +gcc_warn_check +: # guarantee successful exit + +# vim:ft=sh diff --git a/bin/install-qa-check.d/90world-writable b/bin/install-qa-check.d/90world-writable new file mode 100644 index 000000000..635612dc8 --- /dev/null +++ b/bin/install-qa-check.d/90world-writable @@ -0,0 +1,27 @@ +# Check for world-writable files + +world_writable_check() { + # Now we look for all world writable files. + # PREFIX LOCAL: keep offset prefix in the reported files + local unsafe_files=$(find "${ED}" -type f -perm -2 | sed -e "s:^${D}:- :") + # END PREFIX LOCAL + if [[ -n ${unsafe_files} ]] ; then + __vecho "QA Security Notice: world writable file(s):" + __vecho "${unsafe_files}" + __vecho "- This may or may not be a security problem, most of the time it is one." + __vecho "- Please double check that $PF really needs a world writeable bit and file bugs accordingly." + sleep 1 + fi + + local unsafe_files=$(find "${ED}" -type f '(' -perm -2002 -o -perm -4002 ')' | sed -e "s:^${D}:/:") + if [[ -n ${unsafe_files} ]] ; then + eqawarn "QA Notice: Unsafe files detected (set*id and world writable)" + eqawarn "${unsafe_files}" + die "Unsafe files found in \${D}. Portage will not install them." + fi +} + +world_writable_check +: # guarantee successful exit + +# vim:ft=sh diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh index d92103f4f..1904c258f 100644 --- a/bin/misc-functions.sh +++ b/bin/misc-functions.sh @@ -1,5 +1,5 @@ #!@PORTAGE_BASH@ -# Copyright 1999-2013 Gentoo Foundation +# Copyright 1999-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # # Miscellaneous shell functions that make use of the ebuild env but don't need @@ -176,63 +176,32 @@ install_qa_check() { cd "${D}" || die "cd failed" # END PREFIX LOCAL - qa_var="QA_FLAGS_IGNORED_${ARCH/-/_}" - eval "[[ -n \${!qa_var} ]] && QA_FLAGS_IGNORED=(\"\${${qa_var}[@]}\")" - if [[ ${#QA_FLAGS_IGNORED[@]} -eq 1 ]] ; then - local shopts=$- - set -o noglob - QA_FLAGS_IGNORED=(${QA_FLAGS_IGNORED}) - set +o noglob - set -${shopts} - fi + # Run QA checks from install-qa-check.d. + # Note: checks need to be run *before* stripping. + local f + # TODO: handle nullglob-like + for f in "${PORTAGE_BIN_PATH}"/install-qa-check.d/*; do + # Run in a subshell to treat it like external script, + # but use 'source' to pass all variables through. + ( + source "${f}" || eerror "Post-install QA check ${f##*/} failed to run" + ) + done - # Check for files built without respecting *FLAGS. Note that - # -frecord-gcc-switches must be in all *FLAGS variables, in - # order to avoid false positive results here. - # NOTE: This check must execute before prepall/prepstrip, since - # prepstrip strips the .GCC.command.line sections. - if type -P scanelf > /dev/null && ! has binchecks ${RESTRICT} && \ - [[ "${CFLAGS}" == *-frecord-gcc-switches* ]] && \ - [[ "${CXXFLAGS}" == *-frecord-gcc-switches* ]] && \ - [[ "${FFLAGS}" == *-frecord-gcc-switches* ]] && \ - [[ "${FCFLAGS}" == *-frecord-gcc-switches* ]] ; then - rm -f "${T}"/scanelf-ignored-CFLAGS.log - for x in $(scanelf -qyRF '#k%p' -k '!.GCC.command.line' "${ED}") ; do - # Separate out file types that are known to support - # .GCC.command.line sections, using the `file` command - # similar to how prepstrip uses it. - f=$(file "${x}") || continue - [[ -z ${f} ]] && continue - if [[ ${f} == *"SB executable"* || - ${f} == *"SB shared object"* ]] ; then - echo "${x}" >> "${T}"/scanelf-ignored-CFLAGS.log + # Run QA checks from repositories + # (yes, PORTAGE_ECLASS_LOCATIONS contains repo paths...) + local repo_location + for repo_location in "${PORTAGE_ECLASS_LOCATIONS[@]}"; do + for f in "${repo_location}"/metadata/install-qa-check.d/*; do + if [[ -f ${f} ]]; then + ( + # allow inheriting eclasses + _IN_INSTALL_QA_CHECK=1 + source "${f}" || eerror "Post-install QA check ${f##*/} failed to run" + ) fi done - - if [[ -f "${T}"/scanelf-ignored-CFLAGS.log ]] ; then - - if [ "${QA_STRICT_FLAGS_IGNORED-unset}" = unset ] ; then - for x in "${QA_FLAGS_IGNORED[@]}" ; do - sed -e "s#^${x#/}\$##" -i "${T}"/scanelf-ignored-CFLAGS.log - done - fi - # Filter anything under /usr/lib/debug/ in order to avoid - # duplicate warnings for splitdebug files. - sed -e "s#^usr/lib/debug/.*##" -e "/^\$/d" -e "s#^#/#" \ - -i "${T}"/scanelf-ignored-CFLAGS.log - f=$(<"${T}"/scanelf-ignored-CFLAGS.log) - if [[ -n ${f} ]] ; then - __vecho -ne '\n' - eqawarn "${BAD}QA Notice: Files built without respecting CFLAGS have been detected${NORMAL}" - eqawarn " Please include the following list of files in your report:" - eqawarn "${f}" - __vecho -ne '\n' - sleep 1 - else - rm -f "${T}"/scanelf-ignored-CFLAGS.log - fi - fi - fi + done export STRIP_MASK prepall @@ -240,62 +209,6 @@ install_qa_check() { ecompressdir --dequeue ecompress --dequeue - # Prefix specific checks - [[ ${ED} != ${D} ]] && install_qa_check_prefix - - f= - for x in etc/app-defaults usr/man usr/info usr/X11R6 usr/doc usr/locale ; do - [[ -d ${ED}/$x ]] && f+=" $x\n" - done - if [[ -n $f ]] ; then - eqawarn "QA Notice: This ebuild installs into the following deprecated directories:" - eqawarn - eqawarn "$f" - fi - - # It's ok create these directories, but not to install into them. #493154 - # TODO: We should add var/lib to this list. - f= - for x in var/cache var/lock var/run run ; do - if [[ ! -L ${ED}/${x} && -d ${ED}/${x} ]] ; then - if [[ -z $(find "${ED}/${x}" -prune -empty) ]] ; then - f+=$(cd "${ED}"; find "${x}" -printf ' %p\n') - fi - fi - done - if [[ -n ${f} ]] ; then - eqawarn "QA Notice: This ebuild installs into paths that should be created at runtime." - eqawarn " To fix, simply do not install into these directories. Instead, your package" - eqawarn " should create dirs on the fly at runtime as needed via init scripts/etc..." - eqawarn - eqawarn "${f}" - fi - - set +f - f= - for x in "${ED}etc/udev/rules.d/"* "${ED}lib"*"/udev/rules.d/"* ; do - [[ -e ${x} ]] || continue - [[ ${x} == ${ED}lib/udev/rules.d/* ]] && continue - f+=" ${x#${ED}}\n" - done - if [[ -n $f ]] ; then - eqawarn "QA Notice: udev rules should be installed in /lib/udev/rules.d:" - eqawarn - eqawarn "$f" - fi - - # Now we look for all world writable files. - # PREFIX LOCAL: keep offset in the paths - local unsafe_files=$(find "${ED}" -type f -perm -2 | sed -e "s:^${D}:- :") - # END PREFIX LOCAL - if [[ -n ${unsafe_files} ]] ; then - __vecho "QA Security Notice: world writable file(s):" - __vecho "${unsafe_files}" - __vecho "- This may or may not be a security problem, most of the time it is one." - __vecho "- Please double check that $PF really needs a world writeable bit and file bugs accordingly." - sleep 1 - fi - # PREFIX LOCAL: # anything outside the prefix should be caught by the Prefix QA # check, so if there's nothing in ED, we skip searching for QA @@ -329,238 +242,6 @@ install_qa_check() { } install_qa_check_elf() { - if type -P scanelf > /dev/null && ! has binchecks ${RESTRICT}; then - local insecure_rpath=0 tmp_quiet=${PORTAGE_QUIET} - local x - - # display warnings when using stricter because we die afterwards - if has stricter ${FEATURES} ; then - unset PORTAGE_QUIET - fi - - # Make sure we disallow insecure RUNPATH/RPATHs. - # 1) References to PORTAGE_BUILDDIR are banned because it's a - # security risk. We don't want to load files from a - # temporary directory. - # 2) If ROOT != "/", references to ROOT are banned because - # that directory won't exist on the target system. - # 3) Null paths are banned because the loader will search $PWD when - # it finds null paths. - local forbidden_dirs="${PORTAGE_BUILDDIR}" - if [[ -n "${ROOT}" && "${ROOT}" != "/" ]]; then - forbidden_dirs+=" ${ROOT}" - fi - local dir l rpath_files=$(scanelf -F '%F:%r' -qBR "${ED}") - f="" - for dir in ${forbidden_dirs}; do - for l in $(echo "${rpath_files}" | grep -E ":${dir}|::|: "); do - f+=" ${l%%:*}\n" - if ! has stricter ${FEATURES}; then - __vecho "Auto fixing rpaths for ${l%%:*}" - TMPDIR="${dir}" scanelf -BXr "${l%%:*}" -o /dev/null - fi - done - done - - # Reject set*id binaries with $ORIGIN in RPATH #260331 - x=$( - find "${ED}" -type f \( -perm -u+s -o -perm -g+s \) -print0 | \ - xargs -0 scanelf -qyRF '%r %p' | grep '$ORIGIN' - ) - - # Print QA notice. - if [[ -n ${f}${x} ]] ; then - __vecho -ne '\n' - eqawarn "QA Notice: The following files contain insecure RUNPATHs" - eqawarn " Please file a bug about this at http://bugs.gentoo.org/" - eqawarn " with the maintaining herd of the package." - eqawarn "${f}${f:+${x:+\n}}${x}" - __vecho -ne '\n' - if [[ -n ${x} ]] || has stricter ${FEATURES} ; then - insecure_rpath=1 - fi - fi - - # TEXTRELs are baaaaaaaad - # Allow devs to mark things as ignorable ... e.g. things that are - # binary-only and upstream isn't cooperating (nvidia-glx) ... we - # allow ebuild authors to set QA_TEXTRELS_arch and QA_TEXTRELS ... - # the former overrides the latter ... regexes allowed ! :) - qa_var="QA_TEXTRELS_${ARCH/-/_}" - [[ -n ${!qa_var} ]] && QA_TEXTRELS=${!qa_var} - [[ -n ${QA_STRICT_TEXTRELS} ]] && QA_TEXTRELS="" - export QA_TEXTRELS="${QA_TEXTRELS} lib*/modules/*.ko" - f=$(scanelf -qyRF '%t %p' "${ED}" | grep -v 'usr/lib/debug/') - if [[ -n ${f} ]] ; then - scanelf -qyRAF '%T %p' "${PORTAGE_BUILDDIR}"/ &> "${T}"/scanelf-textrel.log - __vecho -ne '\n' - eqawarn "QA Notice: The following files contain runtime text relocations" - eqawarn " Text relocations force the dynamic linker to perform extra" - eqawarn " work at startup, waste system resources, and may pose a security" - eqawarn " risk. On some architectures, the code may not even function" - eqawarn " properly, if at all." - eqawarn " For more information, see http://hardened.gentoo.org/pic-fix-guide.xml" - eqawarn " Please include the following list of files in your report:" - eqawarn "${f}" - __vecho -ne '\n' - die_msg="${die_msg} textrels," - sleep 1 - fi - - # Also, executable stacks only matter on linux (and just glibc atm ...) - f="" - case ${CTARGET:-${CHOST}} in - *-linux-gnu*) - # Check for files with executable stacks, but only on arches which - # are supported at the moment. Keep this list in sync with - # http://www.gentoo.org/proj/en/hardened/gnu-stack.xml (Arch Status) - case ${CTARGET:-${CHOST}} in - arm*|i?86*|ia64*|m68k*|s390*|sh*|x86_64*) - # Allow devs to mark things as ignorable ... e.g. things - # that are binary-only and upstream isn't cooperating ... - # we allow ebuild authors to set QA_EXECSTACK_arch and - # QA_EXECSTACK ... the former overrides the latter ... - # regexes allowed ! :) - - qa_var="QA_EXECSTACK_${ARCH/-/_}" - [[ -n ${!qa_var} ]] && QA_EXECSTACK=${!qa_var} - [[ -n ${QA_STRICT_EXECSTACK} ]] && QA_EXECSTACK="" - qa_var="QA_WX_LOAD_${ARCH/-/_}" - [[ -n ${!qa_var} ]] && QA_WX_LOAD=${!qa_var} - [[ -n ${QA_STRICT_WX_LOAD} ]] && QA_WX_LOAD="" - export QA_EXECSTACK="${QA_EXECSTACK} lib*/modules/*.ko" - export QA_WX_LOAD="${QA_WX_LOAD} lib*/modules/*.ko" - f=$(scanelf -qyRAF '%e %p' "${ED}" | grep -v 'usr/lib/debug/') - ;; - esac - ;; - esac - if [[ -n ${f} ]] ; then - # One more pass to help devs track down the source - scanelf -qyRAF '%e %p' "${PORTAGE_BUILDDIR}"/ &> "${T}"/scanelf-execstack.log - __vecho -ne '\n' - eqawarn "QA Notice: The following files contain writable and executable sections" - eqawarn " Files with such sections will not work properly (or at all!) on some" - eqawarn " architectures/operating systems. A bug should be filed at" - eqawarn " http://bugs.gentoo.org/ to make sure the issue is fixed." - eqawarn " For more information, see http://hardened.gentoo.org/gnu-stack.xml" - eqawarn " Please include the following list of files in your report:" - eqawarn " Note: Bugs should be filed for the respective maintainers" - eqawarn " of the package in question and not hardened@g.o." - eqawarn "${f}" - __vecho -ne '\n' - die_msg="${die_msg} execstacks" - sleep 1 - fi - - # Check for files built without respecting LDFLAGS - if [[ "${LDFLAGS}" == *,--hash-style=gnu* ]] && \ - ! has binchecks ${RESTRICT} ; then - f=$(scanelf -qyRF '#k%p' -k .hash "${ED}") - if [[ -n ${f} ]] ; then - echo "${f}" > "${T}"/scanelf-ignored-LDFLAGS.log - if [ "${QA_STRICT_FLAGS_IGNORED-unset}" = unset ] ; then - for x in "${QA_FLAGS_IGNORED[@]}" ; do - sed -e "s#^${x#/}\$##" -i "${T}"/scanelf-ignored-LDFLAGS.log - done - fi - # Filter anything under /usr/lib/debug/ in order to avoid - # duplicate warnings for splitdebug files. - sed -e "s#^usr/lib/debug/.*##" -e "/^\$/d" -e "s#^#/#" \ - -i "${T}"/scanelf-ignored-LDFLAGS.log - f=$(<"${T}"/scanelf-ignored-LDFLAGS.log) - if [[ -n ${f} ]] ; then - __vecho -ne '\n' - eqawarn "${BAD}QA Notice: Files built without respecting LDFLAGS have been detected${NORMAL}" - eqawarn " Please include the following list of files in your report:" - eqawarn "${f}" - __vecho -ne '\n' - sleep 1 - else - rm -f "${T}"/scanelf-ignored-LDFLAGS.log - fi - fi - fi - - if [[ ${insecure_rpath} -eq 1 ]] ; then - die "Aborting due to serious QA concerns with RUNPATH/RPATH" - elif [[ -n ${die_msg} ]] && has stricter ${FEATURES} ; then - die "Aborting due to QA concerns: ${die_msg}" - fi - - # Check for shared libraries lacking SONAMEs - qa_var="QA_SONAME_${ARCH/-/_}" - eval "[[ -n \${!qa_var} ]] && QA_SONAME=(\"\${${qa_var}[@]}\")" - f=$(scanelf -ByF '%S %p' "${ED}"{,usr/}lib*/lib*.so* | awk '$2 == "" { print }' | sed -e "s:^[[:space:]]${ED}:/:") - if [[ -n ${f} ]] ; then - echo "${f}" > "${T}"/scanelf-missing-SONAME.log - if [[ "${QA_STRICT_SONAME-unset}" == unset ]] ; then - if [[ ${#QA_SONAME[@]} -gt 1 ]] ; then - for x in "${QA_SONAME[@]}" ; do - sed -e "s#^/${x#/}\$##" -i "${T}"/scanelf-missing-SONAME.log - done - else - local shopts=$- - set -o noglob - for x in ${QA_SONAME} ; do - sed -e "s#^/${x#/}\$##" -i "${T}"/scanelf-missing-SONAME.log - done - set +o noglob - set -${shopts} - fi - fi - sed -e "/^\$/d" -i "${T}"/scanelf-missing-SONAME.log - f=$(<"${T}"/scanelf-missing-SONAME.log) - if [[ -n ${f} ]] ; then - __vecho -ne '\n' - eqawarn "QA Notice: The following shared libraries lack a SONAME" - eqawarn "${f}" - __vecho -ne '\n' - sleep 1 - else - rm -f "${T}"/scanelf-missing-SONAME.log - fi - fi - - # Check for shared libraries lacking NEEDED entries - qa_var="QA_DT_NEEDED_${ARCH/-/_}" - eval "[[ -n \${!qa_var} ]] && QA_DT_NEEDED=(\"\${${qa_var}[@]}\")" - # PREFIX LOCAL: keep offset prefix in the recorded files - f=$(scanelf -ByF '%n %p' "${ED}"{,usr/}lib*/lib*.so* | awk '$2 == "" { print }' | sed -e "s:^[[:space:]]${D}:/:") - # END PREFIX LOCAL - if [[ -n ${f} ]] ; then - echo "${f}" > "${T}"/scanelf-missing-NEEDED.log - if [[ "${QA_STRICT_DT_NEEDED-unset}" == unset ]] ; then - if [[ ${#QA_DT_NEEDED[@]} -gt 1 ]] ; then - for x in "${QA_DT_NEEDED[@]}" ; do - sed -e "s#^/${x#/}\$##" -i "${T}"/scanelf-missing-NEEDED.log - done - else - local shopts=$- - set -o noglob - for x in ${QA_DT_NEEDED} ; do - sed -e "s#^/${x#/}\$##" -i "${T}"/scanelf-missing-NEEDED.log - done - set +o noglob - set -${shopts} - fi - fi - sed -e "/^\$/d" -i "${T}"/scanelf-missing-NEEDED.log - f=$(<"${T}"/scanelf-missing-NEEDED.log) - if [[ -n ${f} ]] ; then - __vecho -ne '\n' - eqawarn "QA Notice: The following shared libraries lack NEEDED entries" - eqawarn "${f}" - __vecho -ne '\n' - sleep 1 - else - rm -f "${T}"/scanelf-missing-NEEDED.log - fi - fi - - PORTAGE_QUIET=${tmp_quiet} - fi - # Create NEEDED.ELF.2 regardless of RESTRICT=binchecks, since this info is # too useful not to have (it's required for things like preserve-libs), and # it's tempting for ebuild authors to set RESTRICT=binchecks for packages @@ -591,441 +272,8 @@ install_qa_check_elf() { } install_qa_check_misc() { - # PREFIX LOCAL: keep offset prefix in the reported files - local unsafe_files=$(find "${ED}" -type f '(' -perm -2002 -o -perm -4002 ')' | sed -e "s:^${D}:/:") - # END PREFIX LOCAL - if [[ -n ${unsafe_files} ]] ; then - eqawarn "QA Notice: Unsafe files detected (set*id and world writable)" - eqawarn "${unsafe_files}" - die "Unsafe files found in \${D}. Portage will not install them." - fi - - if [[ -d ${D%/}${D} ]] ; then - local -i INSTALLTOD=0 - while read -r -d $'\0' i ; do - eqawarn "QA Notice: /${i##${D%/}${D}} installed in \${D}/\${D}" - ((INSTALLTOD++)) - done < <(find "${D%/}${D}" -print0) - die "Aborting due to QA concerns: ${INSTALLTOD} files installed in ${D%/}${D}" - fi - - # Sanity check syntax errors in init.d scripts - local d - for d in /etc/conf.d /etc/init.d ; do - [[ -d ${ED}/${d} ]] || continue - for i in "${ED}"/${d}/* ; do - [[ -L ${i} ]] && continue - # if empty conf.d/init.d dir exists (baselayout), then i will be "/etc/conf.d/*" and not exist - [[ ! -e ${i} ]] && continue - if [[ ${d} == /etc/init.d && ${i} != *.sh ]] ; then - # skip non-shell-script for bug #451386 - [[ $(head -n1 "${i}") =~ ^#!.*[[:space:]/](runscript|sh)$ ]] || continue - fi - bash -n "${i}" || die "The init.d file has syntax errors: ${i}" - done - done - - local checkbashisms=$(type -P checkbashisms) - if [[ -n ${checkbashisms} ]] ; then - for d in /etc/init.d ; do - [[ -d ${ED}${d} ]] || continue - for i in "${ED}${d}"/* ; do - [[ -e ${i} ]] || continue - [[ -L ${i} ]] && continue - f=$("${checkbashisms}" -f "${i}" 2>&1) - [[ $? != 0 && -n ${f} ]] || continue - eqawarn "QA Notice: shell script appears to use non-POSIX feature(s):" - while read -r ; - do eqawarn " ${REPLY}" - done <<< "${f//${ED}}" - done - done - fi - - # Look for leaking LDFLAGS into pkg-config files - f=$(egrep -sH '^Libs.*-Wl,(-O[012]|--hash-style)' "${ED}"/usr/*/pkgconfig/*.pc) - if [[ -n ${f} ]] ; then - eqawarn "QA Notice: pkg-config files with wrong LDFLAGS detected:" - eqawarn "${f//${D}}" - fi - - # this should help to ensure that all (most?) shared libraries are executable - # and that all libtool scripts / static libraries are not executable - local j - for i in "${ED}"opt/*/lib* \ - "${ED}"lib* \ - "${ED}"usr/lib* ; do - [[ ! -d ${i} ]] && continue - - for j in "${i}"/*.so.* "${i}"/*.so "${i}"/*.dylib "${i}"/*.dll ; do - [[ ! -e ${j} ]] && continue - [[ -L ${j} ]] && continue - [[ -x ${j} ]] && continue - __vecho "making executable: ${j#${ED}}" - chmod +x "${j}" - done - - for j in "${i}"/*.a "${i}"/*.la ; do - [[ ! -e ${j} ]] && continue - [[ -L ${j} ]] && continue - [[ ! -x ${j} ]] && continue - __vecho "removing executable bit: ${j#${ED}}" - chmod -x "${j}" - done - - for j in "${i}"/*.{a,dll,dylib,sl,so}.* "${i}"/*.{a,dll,dylib,sl,so} ; do - [[ ! -e ${j} ]] && continue - [[ ! -L ${j} ]] && continue - linkdest=$(readlink "${j}") - if [[ ${linkdest} == /* ]] ; then - __vecho -ne '\n' - eqawarn "QA Notice: Found an absolute symlink in a library directory:" - eqawarn " ${j#${D}} -> ${linkdest}" - eqawarn " It should be a relative symlink if in the same directory" - eqawarn " or a linker script if it crosses the /usr boundary." - fi - done - done - - # When installing static libraries into /usr/lib and shared libraries into - # /lib, we have to make sure we have a linker script in /usr/lib along side - # the static library, or gcc will utilize the static lib when linking :(. - # http://bugs.gentoo.org/4411 - abort="no" - local a s - for a in "${ED}"usr/lib*/*.a ; do - # PREFIX LOCAL: support MachO objects - [[ ${CHOST} == *-darwin* ]] \ - && s=${a%.a}.dylib \ - || s=${a%.a}.so - # END PREFIX LOCAL - if [[ ! -e ${s} ]] ; then - s=${s%usr/*}${s##*/usr/} - if [[ -e ${s} ]] ; then - __vecho -ne '\n' - eqawarn "QA Notice: Missing gen_usr_ldscript for ${s##*/}" - abort="yes" - fi - fi - done - [[ ${abort} == "yes" ]] && die "add those ldscripts" - - # Make sure people don't store libtool files or static libs in /lib - # PREFIX LOCAL: on AIX, "dynamic libs" have extension .a, so don't - # get false positives - [[ ${CHOST} == *-aix* ]] \ - && f=$(ls "${ED}"lib*/*.la 2>/dev/null || true) \ - || f=$(ls "${ED}"lib*/*.{a,la} 2>/dev/null) - # END PREFIX LOCAL - if [[ -n ${f} ]] ; then - __vecho -ne '\n' - eqawarn "QA Notice: Excessive files found in the / partition" - eqawarn "${f}" - __vecho -ne '\n' - die "static archives (*.a) and libtool library files (*.la) belong in /usr/lib*, not /lib*" - fi - - # Verify that the libtool files don't contain bogus $D entries. - local abort=no gentoo_bug=no always_overflow=no - for a in "${ED}"usr/lib*/*.la ; do - s=${a##*/} - if grep -qs "${ED}" "${a}" ; then - __vecho -ne '\n' - eqawarn "QA Notice: ${s} appears to contain PORTAGE_TMPDIR paths" - abort="yes" - fi - done - [[ ${abort} == "yes" ]] && die "soiled libtool library files found" - - # Evaluate misc gcc warnings - if [[ -n ${PORTAGE_LOG_FILE} && -r ${PORTAGE_LOG_FILE} ]] ; then - # In debug mode, this variable definition and corresponding grep calls - # will produce false positives if they're shown in the trace. - local reset_debug=0 - if [[ ${-/x/} != $- ]] ; then - set +x - reset_debug=1 - fi - local m msgs=( - ": warning: dereferencing type-punned pointer will break strict-aliasing rules" - ": warning: dereferencing pointer .* does break strict-aliasing rules" - ": warning: implicit declaration of function" - ": warning: incompatible implicit declaration of built-in function" - ": warning: is used uninitialized in this function" # we'll ignore "may" and "might" - ": warning: comparisons like X<=Y<=Z do not have their mathematical meaning" - ": warning: null argument where non-null required" - ": warning: array subscript is below array bounds" - ": warning: array subscript is above array bounds" - ": warning: attempt to free a non-heap object" - ": warning: .* called with .*bigger.* than .* destination buffer" - ": warning: call to .* will always overflow destination buffer" - ": warning: assuming pointer wraparound does not occur when comparing" - ": warning: hex escape sequence out of range" - ": warning: [^ ]*-hand operand of comma .*has no effect" - ": warning: converting to non-pointer type .* from NULL" - ": warning: NULL used in arithmetic" - ": warning: passing NULL to non-pointer argument" - ": warning: the address of [^ ]* will always evaluate as" - ": warning: the address of [^ ]* will never be NULL" - ": warning: too few arguments for format" - ": warning: reference to local variable .* returned" - ": warning: returning reference to temporary" - ": warning: function returns address of local variable" - ": warning: .*\\[-Wsizeof-pointer-memaccess\\]" - ": warning: .*\\[-Waggressive-loop-optimizations\\]" - # this may be valid code :/ - #": warning: multi-character character constant" - # need to check these two ... - #": warning: assuming signed overflow does not occur when" - #": warning: comparison with string literal results in unspecified behav" - # yacc/lex likes to trigger this one - #": warning: extra tokens at end of .* directive" - # only gcc itself triggers this ? - #": warning: .*noreturn.* function does return" - # these throw false positives when 0 is used instead of NULL - #": warning: missing sentinel in function call" - #": warning: not enough variable arguments to fit a sentinel" - ) - abort="no" - i=0 - local grep_cmd=grep - [[ $PORTAGE_LOG_FILE = *.gz ]] && grep_cmd=zgrep - while [[ -n ${msgs[${i}]} ]] ; do - m=${msgs[$((i++))]} - # force C locale to work around slow unicode locales #160234 - f=$(LC_ALL=C $grep_cmd "${m}" "${PORTAGE_LOG_FILE}") - if [[ -n ${f} ]] ; then - abort="yes" - # for now, don't make this fatal (see bug #337031) - #case "$m" in - # ": warning: call to .* will always overflow destination buffer") always_overflow=yes ;; - #esac - if [[ $always_overflow = yes ]] ; then - eerror - eerror "QA Notice: Package triggers severe warnings which indicate that it" - eerror " may exhibit random runtime failures." - eerror - eerror "${f}" - eerror - eerror " Please file a bug about this at http://bugs.gentoo.org/" - eerror " with the maintaining herd of the package." - eerror - else - __vecho -ne '\n' - eqawarn "QA Notice: Package triggers severe warnings which indicate that it" - eqawarn " may exhibit random runtime failures." - eqawarn "${f}" - __vecho -ne '\n' - fi - fi - done - local cat_cmd=cat - [[ $PORTAGE_LOG_FILE = *.gz ]] && cat_cmd=zcat - [[ $reset_debug = 1 ]] && set -x - # Use safe cwd, avoiding unsafe import for bug #469338. - f=$(cd "${PORTAGE_PYM_PATH}" ; $cat_cmd "${PORTAGE_LOG_FILE}" | \ - "${PORTAGE_PYTHON:-@PREFIX_PORTAGE_PYTHON@}" "$PORTAGE_BIN_PATH"/check-implicit-pointer-usage.py || die "check-implicit-pointer-usage.py failed") - if [[ -n ${f} ]] ; then - - # In the future this will be a forced "die". In preparation, - # increase the log level from "qa" to "eerror" so that people - # are aware this is a problem that must be fixed asap. - - # just warn on 32bit hosts but bail on 64bit hosts - case ${CHOST} in - alpha*|hppa64*|ia64*|powerpc64*|mips64*|sparc64*|sparcv9*|x86_64*) gentoo_bug=yes ;; - esac - - abort=yes - - if [[ $gentoo_bug = yes ]] ; then - eerror - eerror "QA Notice: Package triggers severe warnings which indicate that it" - eerror " will almost certainly crash on 64bit architectures." - eerror - eerror "${f}" - eerror - eerror " Please file a bug about this at http://bugs.gentoo.org/" - eerror " with the maintaining herd of the package." - eerror - else - __vecho -ne '\n' - eqawarn "QA Notice: Package triggers severe warnings which indicate that it" - eqawarn " will almost certainly crash on 64bit architectures." - eqawarn "${f}" - __vecho -ne '\n' - fi - - fi - if [[ ${abort} == "yes" ]] ; then - if [[ $gentoo_bug = yes || $always_overflow = yes ]] ; then - die "install aborted due to severe warnings shown above" - else - echo "Please do not file a Gentoo bug and instead" \ - "report the above QA issues directly to the upstream" \ - "developers of this software." | fmt -w 70 | \ - while read -r line ; do eqawarn "${line}" ; done - eqawarn "Homepage: ${HOMEPAGE}" - has stricter ${FEATURES} && \ - die "install aborted due to severe warnings shown above" - fi - fi - fi - # Portage regenerates this on the installed system. rm -f "${ED}"/usr/share/info/dir{,.gz,.bz2} || die "rm failed!" - - if has multilib-strict ${FEATURES} && \ - [[ -x ${EPREFIX}/usr/bin/file && -x ${EPREFIX}/usr/bin/find ]] && \ - [[ -n ${MULTILIB_STRICT_DIRS} && -n ${MULTILIB_STRICT_DENY} ]] - then - rm -f "${T}/multilib-strict.log" - local abort=no dir file - MULTILIB_STRICT_EXEMPT=$(echo ${MULTILIB_STRICT_EXEMPT} | sed -e 's:\([(|)]\):\\\1:g') - for dir in ${MULTILIB_STRICT_DIRS} ; do - [[ -d ${ED}/${dir} ]] || continue - for file in $(find ${ED}/${dir} -type f | grep -v "^${ED}/${dir}/${MULTILIB_STRICT_EXEMPT}"); do - if file ${file} | egrep -q "${MULTILIB_STRICT_DENY}" ; then - echo "${file#${ED}//}" >> "${T}/multilib-strict.log" - fi - done - done - - if [[ -s ${T}/multilib-strict.log ]] ; then - if [[ ${#QA_MULTILIB_PATHS[@]} -eq 1 ]] ; then - local shopts=$- - set -o noglob - QA_MULTILIB_PATHS=(${QA_MULTILIB_PATHS}) - set +o noglob - set -${shopts} - fi - if [ "${QA_STRICT_MULTILIB_PATHS-unset}" = unset ] ; then - for x in "${QA_MULTILIB_PATHS[@]}" ; do - sed -e "s#^${x#/}\$##" -i "${T}/multilib-strict.log" - done - sed -e "/^\$/d" -i "${T}/multilib-strict.log" - fi - if [[ -s ${T}/multilib-strict.log ]] ; then - abort=yes - echo "Files matching a file type that is not allowed:" - while read -r ; do - echo " ${REPLY}" - done < "${T}/multilib-strict.log" - fi - fi - - [[ ${abort} == yes ]] && die "multilib-strict check failed!" - fi -} - -install_qa_check_prefix() { - if [[ -d ${ED%/}/${D} ]] ; then - find "${ED%/}/${D}" | \ - while read i ; do - eqawarn "QA Notice: /${i##${ED%/}/${D}} installed in \${ED}/\${D}" - done - die "Aborting due to QA concerns: files installed in ${ED}/${D}" - fi - - if [[ -d ${ED%/}/${EPREFIX} ]] ; then - find "${ED%/}/${EPREFIX}/" | \ - while read i ; do - eqawarn "QA Notice: ${i#${D}} double prefix" - done - die "Aborting due to QA concerns: double prefix files installed" - fi - - if [[ -d ${D} ]] ; then - INSTALLTOD=$(find ${D%/} | egrep -v "^${ED}" | sed -e "s|^${D%/}||" | awk '{if (length($0) <= length("'"${EPREFIX}"'")) { if (substr("'"${EPREFIX}"'", 1, length($0)) != $0) {print $0;} } else if (substr($0, 1, length("'"${EPREFIX}"'")) != "'"${EPREFIX}"'") {print $0;} }') - if [[ -n ${INSTALLTOD} ]] ; then - eqawarn "QA Notice: the following files are outside of the prefix:" - eqawarn "${INSTALLTOD}" - die "Aborting due to QA concerns: there are files installed outside the prefix" - fi - fi - - # all further checks rely on ${ED} existing - [[ -d ${ED} ]] || return - - # check shebangs, bug #282539 - rm -f "${T}"/non-prefix-shebangs-errs - local WHITELIST=" /usr/bin/env " - # this is hell expensive, but how else? - find "${ED}" -executable \! -type d -print0 \ - | xargs -0 grep -H -n -m1 "^#!" \ - | while read f ; - do - local fn=${f%%:*} - local pos=${f#*:} ; pos=${pos%:*} - local line=${f##*:} - # shebang always appears on the first line ;) - [[ ${pos} != 1 ]] && continue - local oldIFS=${IFS} - IFS=$'\r'$'\n'$'\t'" " - line=( ${line#"#!"} ) - IFS=${oldIFS} - [[ ${WHITELIST} == *" ${line[0]} "* ]] && continue - local fp=${fn#${D}} ; fp=/${fp%/*} - # line[0] can be an absolutised path, bug #342929 - local eprefix=$(canonicalize ${EPREFIX}) - local rf=${fn} - # in case we deal with a symlink, make sure we don't replace it - # with a real file (sed -i does that) - if [[ -L ${fn} ]] ; then - rf=$(readlink ${fn}) - [[ ${rf} != /* ]] && rf=${fn%/*}/${rf} - # ignore symlinks pointing to outside prefix - # as seen in sys-devel/native-cctools - [[ $(canonicalize "/${rf#${D}}") != ${eprefix}/* ]] && continue - fi - # does the shebang start with ${EPREFIX}, and does it exist? - if [[ ${line[0]} == ${EPREFIX}/* || ${line[0]} == ${eprefix}/* ]] ; then - if [[ ! -e ${ROOT%/}${line[0]} && ! -e ${D%/}${line[0]} ]] ; then - # hmm, refers explicitly to $EPREFIX, but doesn't exist, - # if it's in PATH that's wrong in any case - if [[ ":${PATH}:" == *":${fp}:"* ]] ; then - echo "${fn#${D}}:${line[0]} (explicit EPREFIX but target not found)" \ - >> "${T}"/non-prefix-shebangs-errs - else - eqawarn "${fn#${D}} has explicit EPREFIX in shebang but target not found (${line[0]})" - fi - fi - continue - fi - # unprefixed shebang, is the script directly in $PATH or an init - # script? - if [[ ":${PATH}:${EPREFIX}/etc/init.d:" == *":${fp}:"* ]] ; then - if [[ -e ${EROOT}${line[0]} || -e ${ED}${line[0]} ]] ; then - # is it unprefixed, but we can just fix it because a - # prefixed variant exists - eqawarn "prefixing shebang of ${fn#${D}}" - # statement is made idempotent on purpose, because - # symlinks may point to the same target, and hence the - # same real file may be sedded multiple times since we - # read the shebangs in one go upfront for performance - # reasons - sed -i -e '1s:^#! \?'"${line[0]}"':#!'"${EPREFIX}"${line[0]}':' "${rf}" - continue - else - # this is definitely wrong: script in $PATH and invalid shebang - echo "${fn#${D}}:${line[0]} (script ${fn##*/} installed in PATH but interpreter ${line[0]} not found)" \ - >> "${T}"/non-prefix-shebangs-errs - fi - else - # unprefixed/invalid shebang, but outside $PATH, this may be - # intended (e.g. config.guess) so remain silent by default - has stricter ${FEATURES} && \ - eqawarn "invalid shebang in ${fn#${D}}: ${line[0]}" - fi - done - if [[ -e "${T}"/non-prefix-shebangs-errs ]] ; then - eqawarn "QA Notice: the following files use invalid (possible non-prefixed) shebangs:" - while read line ; do - eqawarn " ${line}" - done < "${T}"/non-prefix-shebangs-errs - rm -f "${T}"/non-prefix-shebangs-errs - die "Aborting due to QA concerns: invalid shebangs found" - fi } install_qa_check_macho() { diff --git a/bin/phase-functions.sh b/bin/phase-functions.sh index 042fab53e..2dece5ada 100644 --- a/bin/phase-functions.sh +++ b/bin/phase-functions.sh @@ -382,6 +382,19 @@ __dyn_prepare() { trap - SIGINT SIGQUIT } +# @FUNCTION: __start_distcc +# @DESCRIPTION: +# Start distcc-pump if necessary. +__start_distcc() { + if has distcc $FEATURES && has distcc-pump $FEATURES ; then + if [[ -z $INCLUDE_SERVER_PORT ]] || [[ ! -w $INCLUDE_SERVER_PORT ]] ; then + # adding distcc to PATH repeatedly results in fatal distcc recursion :) + eval $(pump --startup | grep -v PATH) + trap "pump --shutdown >/dev/null" EXIT + fi + fi +} + __dyn_configure() { if [[ -e $PORTAGE_BUILDDIR/.configured ]] ; then @@ -401,6 +414,7 @@ __dyn_configure() { fi trap __abort_configure SIGINT SIGQUIT + __start_distcc __ebuild_phase pre_src_configure @@ -434,13 +448,7 @@ __dyn_compile() { fi trap __abort_compile SIGINT SIGQUIT - - if has distcc $FEATURES && has distcc-pump $FEATURES ; then - if [[ -z $INCLUDE_SERVER_PORT ]] || [[ ! -w $INCLUDE_SERVER_PORT ]] ; then - eval $(pump --startup) - trap "pump --shutdown" EXIT - fi - fi + __start_distcc __ebuild_phase pre_src_compile @@ -464,6 +472,8 @@ __dyn_test() { fi trap "__abort_test" SIGINT SIGQUIT + __start_distcc + if [ -d "${S}" ]; then cd "${S}" else @@ -489,7 +499,11 @@ __dyn_test() { local save_sp=${SANDBOX_PREDICT} addpredict / __ebuild_phase pre_src_test + + __vecho ">>> Test phase: ${CATEGORY}/${PF}" __ebuild_phase src_test + __vecho ">>> Completed testing ${CATEGORY}/${PF}" + >> "$PORTAGE_BUILDDIR/.tested" || \ die "Failed to create $PORTAGE_BUILDDIR/.tested" __ebuild_phase post_src_test @@ -509,6 +523,8 @@ __dyn_install() { return 0 fi trap "__abort_install" SIGINT SIGQUIT + __start_distcc + __ebuild_phase pre_src_install if ___eapi_has_prefix_variables; then @@ -735,91 +751,84 @@ __ebuild_phase_funcs() { [ $# -ne 2 ] && die "expected exactly 2 args, got $#: $*" local eapi=$1 local phase_func=$2 - local default_phases="pkg_nofetch src_unpack src_prepare src_configure - src_compile src_install src_test" - local x y default_func="" - - for x in pkg_nofetch src_unpack src_test ; do - declare -F $x >/dev/null || \ - eval "$x() { __eapi0_$x \"\$@\" ; }" + local all_phases="src_compile pkg_config src_configure pkg_info + src_install pkg_nofetch pkg_postinst pkg_postrm pkg_preinst + src_prepare pkg_prerm pkg_pretend pkg_setup src_test src_unpack" + local x + + # First, set up the error handlers for default* + for x in ${all_phases} ; do + eval "default_${x}() { + die \"default_${x}() is not supported in EAPI='${eapi}' in phase ${phase_func}\" + }" done + # We can just call the specific handler -- it will either error out + # on invalid phase or run it. + eval "default() { + default_${phase_func} + }" + case "$eapi" in + 0|1) # EAPIs not supporting 'default' - 0|1) + for x in pkg_nofetch src_unpack src_test ; do + declare -F $x >/dev/null || \ + eval "$x() { __eapi0_$x; }" + done if ! declare -F src_compile >/dev/null ; then case "$eapi" in 0) - src_compile() { __eapi0_src_compile "$@" ; } + src_compile() { __eapi0_src_compile; } ;; *) - src_compile() { __eapi1_src_compile "$@" ; } + src_compile() { __eapi1_src_compile; } ;; esac fi - - for x in $default_phases ; do - eval "default_$x() { - die \"default_$x() is not supported with EAPI='$eapi' during phase $phase_func\" - }" - done - - eval "default() { - die \"default() is not supported with EAPI='$eapi' during phase $phase_func\" - }" - ;; - *) - + *) # EAPIs supporting 'default' + + # defaults starting with EAPI 0 + [[ ${phase_func} == pkg_nofetch ]] && \ + default_pkg_nofetch() { __eapi0_pkg_nofetch; } + [[ ${phase_func} == src_unpack ]] && \ + default_src_unpack() { __eapi0_src_unpack; } + [[ ${phase_func} == src_test ]] && \ + default_src_test() { __eapi0_src_test; } + + # defaults starting with EAPI 2 + [[ ${phase_func} == src_prepare ]] && \ + default_src_prepare() { __eapi2_src_prepare; } + [[ ${phase_func} == src_configure ]] && \ + default_src_configure() { __eapi2_src_configure; } + [[ ${phase_func} == src_compile ]] && \ + default_src_compile() { __eapi2_src_compile; } + + # bind supported phases to the defaults + declare -F pkg_nofetch >/dev/null || \ + pkg_nofetch() { default; } + declare -F src_unpack >/dev/null || \ + src_unpack() { default; } + declare -F src_prepare >/dev/null || \ + src_prepare() { default; } declare -F src_configure >/dev/null || \ - src_configure() { __eapi2_src_configure "$@" ; } - + src_configure() { default; } declare -F src_compile >/dev/null || \ - src_compile() { __eapi2_src_compile "$@" ; } - - has $eapi 2 3 || declare -F src_install >/dev/null || \ - src_install() { __eapi4_src_install "$@" ; } - - if has $phase_func $default_phases ; then - - __eapi2_pkg_nofetch () { __eapi0_pkg_nofetch "$@" ; } - __eapi2_src_unpack () { __eapi0_src_unpack "$@" ; } - __eapi2_src_prepare () { true ; } - __eapi2_src_test () { __eapi0_src_test "$@" ; } - __eapi2_src_install () { die "$FUNCNAME is not supported" ; } - - for x in $default_phases ; do - eval "default_$x() { __eapi2_$x \"\$@\" ; }" - done - - eval "default() { __eapi2_$phase_func \"\$@\" ; }" + src_compile() { default; } + declare -F src_test >/dev/null || \ + src_test() { default; } - case "$eapi" in - 2|3) - ;; - *) - eval "default_src_install() { __eapi4_src_install \"\$@\" ; }" - [[ $phase_func = src_install ]] && \ - eval "default() { __eapi4_$phase_func \"\$@\" ; }" - ;; - esac - - else - - for x in $default_phases ; do - eval "default_$x() { - die \"default_$x() is not supported in phase $default_func\" - }" - done - - eval "default() { - die \"default() is not supported with EAPI='$eapi' during phase $phase_func\" - }" + # defaults starting with EAPI 4 + if ! has ${eapi} 2 3; then + [[ ${phase_func} == src_install ]] && \ + default_src_install() { __eapi4_src_install; } + declare -F src_install >/dev/null || \ + src_install() { default; } fi - ;; esac } diff --git a/bin/phase-helpers.sh b/bin/phase-helpers.sh index d2f78aa62..0ef083d32 100644 --- a/bin/phase-helpers.sh +++ b/bin/phase-helpers.sh @@ -521,23 +521,20 @@ econf() { done fi + local conf_args=() if ___eapi_econf_passes_--disable-dependency-tracking || ___eapi_econf_passes_--disable-silent-rules; then local conf_help=$("${ECONF_SOURCE}/configure" --help 2>/dev/null) if ___eapi_econf_passes_--disable-dependency-tracking; then - case "${conf_help}" in - *--disable-dependency-tracking*) - set -- --disable-dependency-tracking "$@" - ;; - esac + if [[ ${conf_help} == *--disable-dependency-tracking* ]]; then + conf_args+=( --disable-dependency-tracking ) + fi fi if ___eapi_econf_passes_--disable-silent-rules; then - case "${conf_help}" in - *--disable-silent-rules*) - set -- --disable-silent-rules "$@" - ;; - esac + if [[ ${conf_help} == *--disable-silent-rules* ]]; then + conf_args+=( --disable-silent-rules ) + fi fi fi @@ -554,7 +551,9 @@ econf() { CONF_PREFIX=${CONF_PREFIX#*=} [[ ${CONF_PREFIX} != /* ]] && CONF_PREFIX="/${CONF_PREFIX}" [[ ${CONF_LIBDIR} != /* ]] && CONF_LIBDIR="/${CONF_LIBDIR}" - set -- --libdir="$(__strip_duplicate_slashes "${CONF_PREFIX}${CONF_LIBDIR}")" "$@" + conf_args+=( + --libdir="$(__strip_duplicate_slashes "${CONF_PREFIX}${CONF_LIBDIR}")" + ) fi # Handle arguments containing quoted whitespace (see bug #457136). @@ -570,6 +569,7 @@ econf() { --datadir="${EPREFIX}"/usr/share \ --sysconfdir="${EPREFIX}"/etc \ --localstatedir="${EPREFIX}"/var/lib \ + "${conf_args[@]}" \ "$@" \ "${EXTRA_ECONF[@]}" __vecho "${ECONF_SOURCE}/configure" "$@" @@ -617,8 +617,8 @@ einstall() { mandir="${ED}usr/share/man" \ sysconfdir="${ED}etc" \ ${LOCAL_EXTRA_EINSTALL} \ - ${MAKEOPTS} ${EXTRA_EMAKE} -j1 \ - "$@" install + ${MAKEOPTS} -j1 \ + "$@" ${EXTRA_EMAKE} install fi ${MAKE:-make} prefix="${ED}usr" \ datadir="${ED}usr/share" \ @@ -627,8 +627,8 @@ einstall() { mandir="${ED}usr/share/man" \ sysconfdir="${ED}etc" \ ${LOCAL_EXTRA_EINSTALL} \ - ${MAKEOPTS} ${EXTRA_EMAKE} -j1 \ - "$@" install || die "einstall failed" + ${MAKEOPTS} -j1 \ + "$@" ${EXTRA_EMAKE} install || die "einstall failed" else die "no Makefile found" fi @@ -667,15 +667,13 @@ __eapi0_src_test() { internal_opts+=" -j1" fi if $emake_cmd ${internal_opts} check -n &> /dev/null; then - __vecho ">>> Test phase [check]: ${CATEGORY}/${PF}" + __vecho "${emake_cmd} ${internal_opts} check" >&2 $emake_cmd ${internal_opts} check || \ die "Make check failed. See above for details." elif $emake_cmd ${internal_opts} test -n &> /dev/null; then - __vecho ">>> Test phase [test]: ${CATEGORY}/${PF}" + __vecho "${emake_cmd} ${internal_opts} test" >&2 $emake_cmd ${internal_opts} test || \ die "Make test failed. See above for details." - else - __vecho ">>> Test phase [none]: ${CATEGORY}/${PF}" fi } @@ -684,6 +682,10 @@ __eapi1_src_compile() { __eapi2_src_compile } +__eapi2_src_prepare() { + : +} + __eapi2_src_configure() { if [[ -x ${ECONF_SOURCE:-.}/configure ]] ; then econf diff --git a/bin/portageq b/bin/portageq index ea9dfde24..7b9e17739 100755 --- a/bin/portageq +++ b/bin/portageq @@ -29,16 +29,16 @@ if os.environ.__contains__("PORTAGE_PYTHONPATH"): else: pym_path = os.path.join(os.path.dirname( os.path.dirname(os.path.realpath(__file__))), "pym") -# Avoid sandbox violations after python upgrade. +# Avoid sandbox violations after Python upgrade. if os.environ.get("SANDBOX_ON") == "1": sandbox_write = os.environ.get("SANDBOX_WRITE", "").split(":") - if pym_path not in sandbox_write: - sandbox_write.append(pym_path) - os.environ["SANDBOX_WRITE"] = \ - ":".join(filter(None, sandbox_write)) - del sandbox_write + for pym_path in pym_paths: + if pym_path not in sandbox_write: + sandbox_write.append(pym_path) + os.environ["SANDBOX_WRITE"] = ":".join(filter(None, sandbox_write)) + del pym_path, sandbox_write +del pym_paths -sys.path.insert(0, pym_path) import portage portage._internal_caller = True from portage import os @@ -66,6 +66,10 @@ def uses_eroot(function): function.uses_eroot = True return function +# global to hold all function docstrings to be used for argparse help. +# Avoids python compilation level 2 optimization troubles. +docstrings = {} + #----------------------------------------------------------------------------- # # To add functionality to this tool, add a function below. @@ -73,10 +77,12 @@ def uses_eroot(function): # The format for functions is: # # def function(argv): -# """<list of options for this function> +# <code> +# +# docstrings['function'] = """<list of options for this function> # <description of the function> # """ -# <code> +# function.__doc__ = docstrings['function'] # # "argv" is an array of the command line parameters provided after the command. # @@ -89,9 +95,6 @@ def uses_eroot(function): @uses_eroot def has_version(argv): - """<eroot> <category/package> - Return code 0 if it's available, 1 otherwise. - """ if (len(argv) < 2): print("ERROR: insufficient parameters!") return 3 @@ -132,12 +135,14 @@ def has_version(argv): noiselevel=-1) return 2 +docstrings['has_version'] = """<eroot> <category/package> + Return code 0 if it's available, 1 otherwise. + """ +has_version.__doc__ = docstrings['has_version'] + @uses_eroot def best_version(argv): - """<eroot> <category/package> - Returns category/package-version (without .ebuild). - """ if (len(argv) < 2): print("ERROR: insufficient parameters!") return 3 @@ -171,12 +176,14 @@ def best_version(argv): except KeyError: return 1 +docstrings['best_version'] = """<eroot> <category/package> + Returns category/package-version (without .ebuild). + """ +best_version.__doc__ = docstrings['best_version'] + @uses_eroot def mass_best_version(argv): - """<eroot> [<category/package>]+ - Returns category/package-version (without .ebuild). - """ if (len(argv) < 2): print("ERROR: insufficient parameters!") return 2 @@ -187,6 +194,11 @@ def mass_best_version(argv): except KeyError: return 1 +docstrings['mass_best_version'] = """<eroot> [<category/package>]+ + Returns category/package-version (without .ebuild). + """ +mass_best_version.__doc__ = docstrings['mass_best_version'] + @uses_eroot def metadata(argv): @@ -215,21 +227,17 @@ def metadata(argv): print("Package not found: '%s'" % pkgspec, file=sys.stderr) return 1 -metadata.__doc__ = """ +docstrings['metadata'] = """ <eroot> <pkgtype> <category/package> [<key>]+ Returns metadata values for the specified package. Available keys: %s """ % ','.join(sorted(x for x in portage.auxdbkeys \ if not x.startswith('UNUSED_'))) +metadata.__doc__ = docstrings['metadata'] @uses_eroot def contents(argv): - """<eroot> <category/package> - List the files that are installed for a given package, with - one file listed on each line. All file names will begin with - <eroot>. - """ if len(argv) != 2: print("ERROR: expected 2 parameters, got %d!" % len(argv)) return 2 @@ -245,16 +253,16 @@ def contents(argv): writemsg_stdout(''.join('%s\n' % x for x in sorted(db.getcontents())), noiselevel=-1) +docstrings['contents'] = """<eroot> <category/package> + List the files that are installed for a given package, with + one file listed on each line. All file names will begin with + <eroot>. + """ +contents.__doc__ = docstrings['contents'] + @uses_eroot def owners(argv): - """<eroot> [<filename>]+ - Given a list of files, print the packages that own the files and which - files belong to each package. Files owned by a package are listed on - the lines below it, indented by a single tab character (\\t). All file - paths must either start with <eroot> or be a basename alone. - Returns 1 if no owners could be found, and 0 otherwise. - """ if len(argv) < 2: sys.stderr.write("ERROR: insufficient parameters!\n") sys.stderr.flush() @@ -325,13 +333,18 @@ def owners(argv): return 0 return 1 +docstrings['owners'] = """<eroot> [<filename>]+ + Given a list of files, print the packages that own the files and which + files belong to each package. Files owned by a package are listed on + the lines below it, indented by a single tab character (\\t). All file + paths must either start with <eroot> or be a basename alone. + Returns 1 if no owners could be found, and 0 otherwise. + """ +owners.__doc__ = docstrings['owners'] + @uses_eroot def is_protected(argv): - """<eroot> <filename> - Given a single filename, return code 0 if it's protected, 1 otherwise. - The filename must begin with <eroot>. - """ if len(argv) != 2: sys.stderr.write("ERROR: expected 2 parameters, got %d!\n" % len(argv)) sys.stderr.flush() @@ -372,13 +385,15 @@ def is_protected(argv): return 0 return 1 +docstrings['is_protected'] = """<eroot> <filename> + Given a single filename, return code 0 if it's protected, 1 otherwise. + The filename must begin with <eroot>. + """ +is_protected.__doc__ = docstrings['is_protected'] + @uses_eroot def filter_protected(argv): - """<eroot> - Read filenames from stdin and write them to stdout if they are protected. - All filenames are delimited by \\n and must begin with <eroot>. - """ if len(argv) != 1: sys.stderr.write("ERROR: expected 1 parameter, got %d!\n" % len(argv)) sys.stderr.flush() @@ -430,14 +445,15 @@ def filter_protected(argv): return 0 +docstrings['filter_protected'] = """<eroot> + Read filenames from stdin and write them to stdout if they are protected. + All filenames are delimited by \\n and must begin with <eroot>. + """ +filter_protected.__doc__ = docstrings['filter_protected'] + @uses_eroot def best_visible(argv): - """<eroot> [pkgtype] <atom> - Returns category/package-version (without .ebuild). - The pkgtype argument defaults to "ebuild" if unspecified, - otherwise it must be one of ebuild, binary, or installed. - """ if (len(argv) < 2): writemsg("ERROR: insufficient parameters!\n", noiselevel=-1) return 2 @@ -512,14 +528,16 @@ def best_visible(argv): return 1 - -@uses_eroot -def mass_best_visible(argv): - """<eroot> [<type>] [<category/package>]+ +docstrings['best_visible'] = """<eroot> [pkgtype] <atom> Returns category/package-version (without .ebuild). The pkgtype argument defaults to "ebuild" if unspecified, otherwise it must be one of ebuild, binary, or installed. """ +best_visible.__doc__ = docstrings['best_visible'] + + +@uses_eroot +def mass_best_visible(argv): type_map = { "ebuild":"porttree", "binary":"bintree", @@ -539,12 +557,16 @@ def mass_best_visible(argv): except KeyError: return 1 +docstrings['mass_best_visible'] = """<eroot> [<type>] [<category/package>]+ + Returns category/package-version (without .ebuild). + The pkgtype argument defaults to "ebuild" if unspecified, + otherwise it must be one of ebuild, binary, or installed. + """ +mass_best_visible.__doc__ = docstrings['mass_best_visible'] + @uses_eroot def all_best_visible(argv): - """<eroot> - Returns all best_visible packages (without .ebuild). - """ if len(argv) < 1: sys.stderr.write("ERROR: insufficient parameters!\n") sys.stderr.flush() @@ -556,14 +578,14 @@ def all_best_visible(argv): if mybest: print(mybest) +docstrings['all_best_visible'] = """<eroot> + Returns all best_visible packages (without .ebuild). + """ +all_best_visible.__doc__ = docstrings['all_best_visible'] + @uses_eroot def match(argv): - """<eroot> <atom> - Returns a \\n separated list of category/package-version. - When given an empty string, all installed packages will - be listed. - """ if len(argv) != 2: print("ERROR: expected 2 parameters, got %d!" % len(argv)) return 2 @@ -605,20 +627,16 @@ def match(argv): for cpv in results: print(cpv) +docstrings['match'] = """<eroot> <atom> + Returns a \\n separated list of category/package-version. + When given an empty string, all installed packages will + be listed. + """ +match.__doc__ = docstrings['match'] + @uses_eroot def expand_virtual(argv): - """<eroot> <atom> - Returns a \\n separated list of atoms expanded from a - given virtual atom (GLEP 37 virtuals only), - excluding blocker atoms. Satisfied - virtual atoms are not included in the output, since - they are expanded to real atoms which are displayed. - Unsatisfied virtual atoms are displayed without - any expansion. The "match" command can be used to - resolve the returned atoms to specific installed - packages. - """ if len(argv) != 2: writemsg("ERROR: expected 2 parameters, got %d!\n" % len(argv), noiselevel=-1) @@ -641,101 +659,134 @@ def expand_virtual(argv): return os.EX_OK +docstrings['expand_virtual'] = """<eroot> <atom> + Returns a \\n separated list of atoms expanded from a + given virtual atom (GLEP 37 virtuals only), + excluding blocker atoms. Satisfied + virtual atoms are not included in the output, since + they are expanded to real atoms which are displayed. + Unsatisfied virtual atoms are displayed without + any expansion. The "match" command can be used to + resolve the returned atoms to specific installed + packages. + """ +expand_virtual.__doc__ = docstrings['expand_virtual'] + def vdb_path(_argv): - """ - Returns the path used for the var(installed) package database for the - set environment/configuration options. - """ out = sys.stdout out.write(os.path.join(portage.settings["EROOT"], portage.VDB_PATH) + "\n") out.flush() return os.EX_OK -def gentoo_mirrors(_argv): +docstrings['vdb_path'] = """ + Returns the path used for the var(installed) package database for the + set environment/configuration options. """ +vdb_path.__doc__ = docstrings['vdb_path'] + + +def gentoo_mirrors(_argv): + print(portage.settings["GENTOO_MIRRORS"]) + +docstrings['gentoo_mirrors'] = """ Returns the mirrors set to use in the portage configuration. """ - print(portage.settings["GENTOO_MIRRORS"]) +gentoo_mirrors.__doc__ = docstrings['gentoo_mirrors'] @uses_eroot def repositories_configuration(argv): - """<eroot> - Returns the configuration of repositories. - """ if len(argv) < 1: print("ERROR: insufficient parameters!", file=sys.stderr) return 3 sys.stdout.write(portage.db[argv[0]]["vartree"].settings.repositories.config_string()) sys.stdout.flush() +docstrings['repositories_configuration'] = """<eroot> + Returns the configuration of repositories. + """ +repositories_configuration.__doc__ = docstrings['repositories_configuration'] + + @uses_eroot def repos_config(argv): - """ + return repositories_configuration(argv) + +docstrings['repos_config'] = """ <eroot> This is an alias for the repositories_configuration command. """ - return repositories_configuration(argv) +repos_config.__doc__ = docstrings['repos_config'] + def portdir(_argv): - """ + print("WARNING: 'portageq portdir' is deprecated. Use 'portageq repositories_configuration' instead.", file=sys.stderr) + print(portage.settings["PORTDIR"]) + +docstrings['portdir'] = """ Returns the PORTDIR path. Deprecated in favor of repositories_configuration command. """ - print("WARNING: 'portageq portdir' is deprecated. Use 'portageq repositories_configuration' instead.", file=sys.stderr) - print(portage.settings["PORTDIR"]) +portdir.__doc__ = docstrings['portdir'] def config_protect(_argv): - """ + print(portage.settings["CONFIG_PROTECT"]) + +docstrings['config_protect'] = """ Returns the CONFIG_PROTECT paths. """ - print(portage.settings["CONFIG_PROTECT"]) +config_protect.__doc__ = docstrings['config_protect'] def config_protect_mask(_argv): - """ - Returns the CONFIG_PROTECT_MASK paths. - """ print(portage.settings["CONFIG_PROTECT_MASK"]) +docstrings['config_protect_mask'] = """ + Returns the CONFIG_PROTECT_MASK paths. + """ +config_protect_mask.__doc__ = docstrings['config_protect_mask'] def portdir_overlay(_argv): - """ + print("WARNING: 'portageq portdir_overlay' is deprecated. Use 'portageq repositories_configuration' instead.", file=sys.stderr) + print(portage.settings["PORTDIR_OVERLAY"]) + +docstrings['portdir_overlay'] = """ Returns the PORTDIR_OVERLAY path. Deprecated in favor of repositories_configuration command. """ - print("WARNING: 'portageq portdir_overlay' is deprecated. Use 'portageq repositories_configuration' instead.", file=sys.stderr) - print(portage.settings["PORTDIR_OVERLAY"]) +portdir_overlay.__doc__ = docstrings['portdir_overlay'] def pkgdir(_argv): - """ + print(portage.settings["PKGDIR"]) + +docstrings['pkgdir'] = """ Returns the PKGDIR path. """ - print(portage.settings["PKGDIR"]) +pkgdir.__doc__ = docstrings['pkgdir'] def distdir(_argv): - """ + print(portage.settings["DISTDIR"]) + +docstrings['distdir'] = """ Returns the DISTDIR path. """ - print(portage.settings["DISTDIR"]) +distdir.__doc__ = docstrings['distdir'] def colormap(_argv): - """ + print(portage.output.colormap()) + +docstrings['colormap'] = """ Display the color.map as environment variables. """ - print(portage.output.colormap()) +colormap.__doc__ = docstrings['colormap'] def envvar(argv): - """<variable>+ - Returns a specific environment variable as exists prior to ebuild.sh. - Similar to: emerge --verbose --info | egrep '^<variable>=' - """ verbose = "-v" in argv if verbose: argv.pop(argv.index("-v")) @@ -752,23 +803,28 @@ def envvar(argv): else: print(portage.settings[arg]) +docstrings['envvar'] = """<variable>+ + Returns a specific environment variable as exists prior to ebuild.sh. + Similar to: emerge --verbose --info | egrep '^<variable>=' + """ +envvar.__doc__ = docstrings['envvar'] + @uses_eroot def get_repos(argv): - """<eroot> - Returns all repos with names (repo_name file) argv[0] = $EROOT - """ if len(argv) < 1: print("ERROR: insufficient parameters!") return 2 print(" ".join(reversed(portage.db[argv[0]]["vartree"].settings.repositories.prepos_order))) +docstrings['get_repos'] = """<eroot> + Returns all repos with names (repo_name file) argv[0] = $EROOT + """ +get_repos.__doc__ = docstrings['get_repos'] + @uses_eroot def master_repositories(argv): - """<eroot> <repo_id>+ - Returns space-separated list of master repositories for specified repository. - """ if len(argv) < 2: print("ERROR: insufficient parameters!", file=sys.stderr) return 3 @@ -784,18 +840,25 @@ def master_repositories(argv): else: print(" ".join(x.name for x in repo.masters)) +docstrings['master_repositories'] = """<eroot> <repo_id>+ + Returns space-separated list of master repositories for specified repository. + """ +master_repositories.__doc__ = docstrings['master_repositories'] + + @uses_eroot def master_repos(argv): - """<eroot> <repo_id>+ + return master_repositories(argv) + +docstrings['master_repos'] = """<eroot> <repo_id>+ This is an alias for the master_repositories command. """ - return master_repositories(argv) +master_repos.__doc__ = docstrings['master_repos'] + @uses_eroot def get_repo_path(argv): - """<eroot> <repo_id>+ - Returns the path to the repo named argv[1], argv[0] = $EROOT - """ + if len(argv) < 2: print("ERROR: insufficient parameters!", file=sys.stderr) return 3 @@ -809,12 +872,14 @@ def get_repo_path(argv): return 1 print(path) +docstrings['get_repo_path'] = """<eroot> <repo_id>+ + Returns the path to the repo named argv[1], argv[0] = $EROOT + """ +get_repo_path.__doc__ = docstrings['get_repo_path'] + @uses_eroot def available_eclasses(argv): - """<eroot> <repo_id>+ - Returns space-separated list of available eclasses for specified repository. - """ if len(argv) < 2: print("ERROR: insufficient parameters!", file=sys.stderr) return 3 @@ -830,12 +895,14 @@ def available_eclasses(argv): else: print(" ".join(sorted(repo.eclass_db.eclasses))) +docstrings['available_eclasses'] = """<eroot> <repo_id>+ + Returns space-separated list of available eclasses for specified repository. + """ +available_eclasses.__doc__ = docstrings['available_eclasses'] + @uses_eroot def eclass_path(argv): - """<eroot> <repo_id> <eclass>+ - Returns the path to specified eclass for specified repository. - """ if len(argv) < 3: print("ERROR: insufficient parameters!", file=sys.stderr) return 3 @@ -859,12 +926,14 @@ def eclass_path(argv): print(eclass.location) return retval +docstrings['eclass_path'] = """<eroot> <repo_id> <eclass>+ + Returns the path to specified eclass for specified repository. + """ +eclass_path.__doc__ = docstrings['eclass_path'] + @uses_eroot def license_path(argv): - """<eroot> <repo_id> <license>+ - Returns the path to specified license for specified repository. - """ if len(argv) < 3: print("ERROR: insufficient parameters!", file=sys.stderr) return 3 @@ -890,15 +959,14 @@ def license_path(argv): print(eclass_path) return retval +docstrings['license_path'] = """<eroot> <repo_id> <license>+ + Returns the path to specified license for specified repository. + """ +license_path.__doc__ = docstrings['license_path'] + @uses_eroot def list_preserved_libs(argv): - """<eroot> - Print a list of libraries preserved during a package update in the form - package: path. Returns 1 if no preserved libraries could be found, - 0 otherwise. - """ - if len(argv) != 1: print("ERROR: wrong number of arguments") return 2 @@ -914,6 +982,13 @@ def list_preserved_libs(argv): writemsg_stdout(''.join(msg), noiselevel=-1) return rValue +docstrings['list_preserved_libs'] = """<eroot> + Print a list of libraries preserved during a package update in the form + package: path. Returns 1 if no preserved libraries could be found, + 0 otherwise. + """ +list_preserved_libs.__doc__ = docstrings['list_preserved_libs'] + class MaintainerEmailMatcher(object): def __init__(self, maintainer_emails): @@ -938,10 +1013,6 @@ class HerdMatcher(object): def pquery(parser, opts, args): - """[options] [atom]+ - Emulates a subset of Pkgcore's pquery tool. - """ - portdb = portage.db[portage.root]['porttree'].dbapi root_config = RootConfig(portdb.settings, portage.db[portage.root], None) @@ -1120,6 +1191,11 @@ def pquery(parser, opts, args): return os.EX_OK +docstrings['pquery'] = """[options] [atom]+ + Emulates a subset of Pkgcore's pquery tool. + """ +pquery.__doc__ = docstrings['pquery'] + #----------------------------------------------------------------------------- # @@ -1217,10 +1293,7 @@ def usage(argv): # help_mode = '--help' in argv for name in commands: - # Drop non-functions - obj = globals()[name] - - doc = obj.__doc__ + doc = docstrings.get(name) if (doc == None): print(" " + name) print(" MISSING DOCUMENTATION!") diff --git a/bin/quickpkg b/bin/quickpkg index d787ca1f3..7019fb351 100755 --- a/bin/quickpkg +++ b/bin/quickpkg @@ -2,7 +2,7 @@ # Copyright 1999-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function +from __future__ import division, print_function import errno import math @@ -11,8 +11,8 @@ import sys import tarfile from os import path as osp -pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") -sys.path.insert(0, pym_path) +if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), ".portage_not_installed")): + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage portage._internal_caller = True from portage import os @@ -264,7 +264,7 @@ def quickpkg_main(options, args, eout): size_str = "0" else: power_of_2 = math.log(size, 2) - power_of_2 = 10*int(power_of_2/10) + power_of_2 = 10*(power_of_2//10) unit = units.get(power_of_2) if unit: size = float(size)/(2**power_of_2) diff --git a/bin/regenworld b/bin/regenworld index d3fb9ed24..1c7678cd3 100755 --- a/bin/regenworld +++ b/bin/regenworld @@ -6,8 +6,8 @@ from __future__ import print_function import sys from os import path as osp -pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") -sys.path.insert(0, pym_path) +if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), ".portage_not_installed")): + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage portage._internal_caller = True from portage import os diff --git a/bin/repoman b/bin/repoman index efab92183..918357747 100755 --- a/bin/repoman +++ b/bin/repoman @@ -32,8 +32,8 @@ except ImportError: from urlparse import urlparse from os import path as osp -pym_path = osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym") -sys.path.insert(0, pym_path) +if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), ".portage_not_installed")): + sys.path.insert(0, osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), "pym")) import portage portage._internal_caller = True portage._disable_legacy_globals() @@ -62,7 +62,7 @@ from repoman import utilities from repoman.herdbase import make_herd_base from _emerge.Package import Package from _emerge.RootConfig import RootConfig -from _emerge.userquery import userquery +from _emerge.UserQuery import UserQuery import portage.checksum import portage.const import portage.repository.config @@ -299,6 +299,7 @@ qahelp = { "dependency.badindev": "User-visible ebuilds with unsatisfied dependencies (matched against *visible* ebuilds) in developing arch", "dependency.badmaskedindev": "Masked ebuilds with unsatisfied dependencies (matched against *all* ebuilds) in developing arch", "dependency.badtilde": "Uses the ~ dep operator with a non-zero revision part, which is useless (the revision is ignored)", + "dependency.perlcore": "This ebuild directly depends on a package in perl-core; it should use the corresponding virtual instead.", "dependency.syntax": "Syntax error in dependency string (usually an extra/missing space/parenthesis)", "dependency.unknown": "Ebuild has a dependency that refers to an unknown package (which may be valid if it is a blocker for a renamed/removed package, or is an alternative choice provided by an overlay)", "file.executable": "Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the executable bit", @@ -318,7 +319,6 @@ qahelp = { "LICENSE.missing": "Ebuilds that have a missing or empty LICENSE variable", "LICENSE.virtual": "Virtuals that have a non-empty LICENSE variable", "DESCRIPTION.missing": "Ebuilds that have a missing or empty DESCRIPTION variable", - "DESCRIPTION.punctuation": "DESCRIPTION ends with a period character", "DESCRIPTION.toolong": "DESCRIPTION is over %d characters" % max_desc_len, "EAPI.definition": "EAPI definition does not conform to PMS section 7.3.1 (first non-comment, non-blank line)", "EAPI.deprecated": "Ebuilds that use features that are deprecated in the current EAPI", @@ -387,7 +387,7 @@ qawarnings = set(( "dependency.badindev", "dependency.badmaskedindev", "dependency.badtilde", -"DESCRIPTION.punctuation", +"dependency.perlcore", "DESCRIPTION.toolong", "EAPI.deprecated", "HOMEPAGE.virtual", @@ -1895,12 +1895,6 @@ for x in effective_scanlist: stats[myqakey] += 1 fails[myqakey].append(relative_path) - if myaux['DESCRIPTION'][-1:] in ['.']: - stats['DESCRIPTION.punctuation'] += 1 - fails['DESCRIPTION.punctuation'].append( - "%s: DESCRIPTION ends with a '%s' character" - % (relative_path, myaux['DESCRIPTION'][-1:])) - # 14 is the length of DESCRIPTION="" if len(myaux['DESCRIPTION']) > max_desc_len: stats['DESCRIPTION.toolong'] += 1 @@ -2042,16 +2036,16 @@ for x in effective_scanlist: if atom == "||": continue + is_blocker = atom.blocker + # Skip dependency.unknown for blockers, so that we # don't encourage people to remove necessary blockers, # as discussed in bug #382407. - if atom.blocker is None and \ + if not is_blocker and \ not portdb.xmatch("match-all", atom) and \ not atom.cp.startswith("virtual/"): unknown_pkgs.add((mytype, atom.unevaluated_atom)) - is_blocker = atom.blocker - if catdir != "virtual": if not is_blocker and \ atom.cp in suspect_virtual: @@ -2060,6 +2054,13 @@ for x in effective_scanlist: relative_path + ": %s: consider using '%s' instead of '%s'" % (mytype, suspect_virtual[atom.cp], atom)) + if not is_blocker and \ + atom.cp.startswith("perl-core/"): + stats['dependency.perlcore'] += 1 + fails['dependency.perlcore'].append( + relative_path + + ": %s: please use '%s' instead of '%s'" % + (mytype, atom.replace("perl-core/","virtual/perl-"), atom)) if buildtime and \ not is_blocker and \ @@ -2882,7 +2883,8 @@ else: logging.info("myupdates: %s", myupdates) logging.info("myheaders: %s", myheaders) - if options.ask and userquery('Commit changes?', True) != 'Yes': + uq = UserQuery(options) + if options.ask and uq.query('Commit changes?', True) != 'Yes': print("* aborting commit.") sys.exit(128 + signal.SIGINT) diff --git a/bin/save-ebuild-env.sh b/bin/save-ebuild-env.sh index 194e46ea9..b790306d1 100644 --- a/bin/save-ebuild-env.sh +++ b/bin/save-ebuild-env.sh @@ -42,7 +42,7 @@ __save_ebuild_env() { for x in pkg_setup pkg_nofetch src_unpack src_prepare src_configure \ src_compile src_test src_install pkg_preinst pkg_postinst \ - pkg_prerm pkg_postrm ; do + pkg_prerm pkg_postrm pkg_config pkg_info pkg_pretend ; do unset -f default_$x __eapi{0,1,2,3,4}_$x done unset x diff --git a/bin/xattr-helper.py b/bin/xattr-helper.py index fdbed54ac..f84e94340 100755 --- a/bin/xattr-helper.py +++ b/bin/xattr-helper.py @@ -2,7 +2,7 @@ # Copyright 2012-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -"""Dump and restore extended attributes. +doc = """Dump and restore extended attributes. We use formats like that used by getfattr --dump. This is meant for shell helpers to save/restore. If you're looking for a python/portage API, see @@ -10,6 +10,8 @@ portage.util.movefile._copyxattr instead. https://en.wikipedia.org/wiki/Extended_file_attributes """ +__doc__ = doc + import array import os @@ -147,7 +149,7 @@ def restore_xattrs(file_in): def main(argv): - parser = ArgumentParser(description=__doc__) + parser = ArgumentParser(description=doc) parser.add_argument('paths', nargs='*', default=[]) actions = parser.add_argument_group('Actions') diff --git a/cnf/sets/portage.conf b/cnf/sets/portage.conf index b73afb19f..fd2c38778 100644 --- a/cnf/sets/portage.conf +++ b/cnf/sets/portage.conf @@ -84,3 +84,8 @@ class = portage.sets.dbapi.UnavailableSet # are not available. [unavailable-binaries] class = portage.sets.dbapi.UnavailableBinaries + +# Installed packages for which vdb *DEPEND entries are outdated compared +# to the matching portdb entry. +[changed-deps] +class = portage.sets.dbapi.ChangedDepsSet diff --git a/doc/Makefile b/doc/Makefile deleted file mode 100644 index 2a981cf46..000000000 --- a/doc/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -SHELL = @PORTAGE_BASH@ - -all: xhtml xhtml-nochunks - -XMLTO_FLAGS = -m custom.xsl -man pdf txt xhtml xhtml-nochunks: - xmlto $@ $(XMLTO_FLAGS) portage.docbook - -clean distclean: - rm -f *.1 *.html portage.txt - -.PHONY: all clean distclean \ - man pdf txt xhtml xhtml-nochunks diff --git a/doc/fragment/version b/doc/fragment/version deleted file mode 100644 index f85674c50..000000000 --- a/doc/fragment/version +++ /dev/null @@ -1 +0,0 @@ -<releaseinfo>VERSION</releaseinfo> diff --git a/man/emerge.1 b/man/emerge.1 index f887bf723..169d34c9b 100644 --- a/man/emerge.1 +++ b/man/emerge.1 @@ -1,4 +1,4 @@ -.TH "EMERGE" "1" "Mar 2014" "Portage VERSION" "Portage" +.TH "EMERGE" "1" "Sep 2014" "Portage VERSION" "Portage" .SH "NAME" emerge \- Command\-line interface to the Portage system .SH "SYNOPSIS" @@ -297,6 +297,20 @@ re\-distributable. With default configuration, this would result in an effective \fBACCEPT_RESTRICT\fR value of "* -bindist". .TP +.BR "\-\-alert [ y | n ] (\-A short option)" +Add a terminal bell character ('\\a') to all interactive prompts. This +is especially useful if dependency resolution is taking a long time, and +you want emerge to alert you when it is finished. If you use +\fBemerge -auAD world\fR, emerge will courteously point out when it has +finished calculating the graph. + +\fB--alert\fR may be 'y' or 'n'. 'true' and 'false' mean the same thing. +Using \fB--alert\fR without an option is the same as using it with 'y'. +Try it with 'emerge -aA portage'. + +If your terminal emulator is set up to make '\\a' into a window manager +urgency hint, move your cursor to a different window to get the effect. +.TP .BR "\-\-alphabetical " When displaying USE and other flag output, combines the enabled and disabled lists into one list and sorts the whole list alphabetically. @@ -353,7 +367,8 @@ to config files, respecting \fBCONFIG_PROTECT\fR and \fB\-\-ask\fR. If the corresponding package.* is a file, the changes are appended to it, if it is a directory, changes are written to the lexicographically last file. This way it is always ensured that the new changes take -precedence over existing changes. +precedence over existing changes. This option is automatically enabled with +\-\-ask. .TP .BR \-\-backtrack=COUNT Specifies an integer number of times to backtrack if diff --git a/man/repoman.1 b/man/repoman.1 index a78f94e90..c57676870 100644 --- a/man/repoman.1 +++ b/man/repoman.1 @@ -12,6 +12,10 @@ checks the quality of ebuild repositories. Note: \fBrepoman commit\fR only works \fIinside local\fR cvs, git, or subversion repositories. + +Note: Messages pertaining to specific lines may be inaccurate in the +prescence of continuation lines from use of the \fI\\\fR character in +BASH. .SH OPTIONS .TP \fB-a\fR, \fB--ask\fR diff --git a/misc/emerge-delta-webrsync b/misc/emerge-delta-webrsync index b31015e93..3ecfc7ff0 100755 --- a/misc/emerge-delta-webrsync +++ b/misc/emerge-delta-webrsync @@ -338,12 +338,12 @@ sync_local() { if has metadata-transfer ${FEATURES} ; then __vecho "Updating cache ..." - "${PORTAGE_BIN_PATH}/emerge" --metadata + emerge --metadata fi local post_sync=${PORTAGE_CONFIGROOT}etc/portage/bin/post_sync [ -x "${post_sync}" ] && "${post_sync}" # --quiet suppresses output if there are no relevant news items - has news ${FEATURES} && "${PORTAGE_BIN_PATH}/emerge" --check-news --quiet + has news ${FEATURES} && emerge --check-news --quiet return 0 } diff --git a/mkrelease.sh b/mkrelease.sh deleted file mode 100755 index f9f75644e..000000000 --- a/mkrelease.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/bin/bash -# Copyright 2008-2014 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -RELEASE_BUILDDIR=${RELEASE_BUILDDIR:-/var/tmp/portage-release} -SOURCE_DIR=${RELEASE_BUILDDIR}/checkout -BRANCH=${BRANCH:-master} -USE_TAG=false -CHANGELOG_REVISION= -UPLOAD_LOCATION= -RUNTESTS=false -USER= - -usage() { - echo "Usage: ${0##*/} [--changelog-rev <tree-ish>] [-t|--tag] [-u|--upload <location>] [--user <username>] [--runtests] <version>" - exit ${1:-0} -} - -die() { - printf 'error: %s\n' "$*" - usage 1 -} - -ARGS=$(getopt -o htu: --long help,changelog-rev:,runtests,tag,upload:,user: \ - -n "${0##*/}" -- "$@") -[ $? != 0 ] && die "initialization error" - -eval set -- "${ARGS}" - -while true; do - case $1 in - --changelog-rev) - CHANGELOG_REVISION=$2 - shift 2 - ;; - -t|--tag) - USE_TAG=true - shift - ;; - -u|--upload) - UPLOAD_LOCATION=$2 - shift 2 - ;; - --user) - USER=$2"@" - shift 2 - ;; - -h|--help) - usage - ;; - --runtests) - RUNTESTS=true - shift - ;; - --) - shift - break - ;; - *) - die "unknown option: $1" - ;; - esac -done - -[ $# != 1 ] && die "Need version argument" -[[ -n ${1/[0-9]*} ]] && die "Invalid version argument" - -VERSION=$1 -RELEASE=portage-${VERSION} -RELEASE_DIR=${RELEASE_BUILDDIR}/${RELEASE} -RELEASE_TARBALL="${RELEASE_BUILDDIR}/${RELEASE}.tar.bz2" -TREE_ISH=${BRANCH} -if [[ ${USE_TAG} == "true" ]] ; then - TREE_ISH="v${VERSION}" -fi - -echo ">>> Cleaning working directories ${RELEASE_DIR} ${SOURCE_DIR}" -rm -rf "${RELEASE_DIR}" "${SOURCE_DIR}" || die "directory cleanup failed" -mkdir -p "${RELEASE_DIR}" || die "directory creation failed" -mkdir -p "${SOURCE_DIR}" || die "mkdir failed" - -echo ">>> Starting GIT archive" -git archive --format=tar ${TREE_ISH} | \ - tar -xf - -C "${SOURCE_DIR}" || die "git archive failed" - -echo ">>> Building release tree" -cp -a "${SOURCE_DIR}/"{bin,cnf,doc,man,misc,pym} "${RELEASE_DIR}/" || die "directory copy failed" -cp "${SOURCE_DIR}/"{.portage_not_installed,DEVELOPING,LICENSE,Makefile,NEWS,README,RELEASE-NOTES,TEST-NOTES} \ - "${RELEASE_DIR}/" || die "file copy failed" - -if [[ ${RUNTESTS} == "true" ]] ; then - pushd "${SOURCE_DIR}" >/dev/null - ./runtests.sh --python-versions=supported || die "tests failed" - popd >/dev/null -fi - -rm -rf "${SOURCE_DIR}" || die "directory cleanup failed" - -echo ">>> Setting portage.VERSION" -sed -e "s/^VERSION = .*/VERSION = \"${VERSION}\"/" \ - -i "${RELEASE_DIR}/pym/portage/__init__.py" || \ - die "Failed to patch portage.VERSION" - -echo ">>> Creating Changelog" -git_log_opts="" -if [[ -n ${CHANGELOG_REVISION} ]] ; then - git_log_opts+=" ${CHANGELOG_REVISION}^..${TREE_ISH}" -else - git_log_opts+=" ${TREE_ISH}" -fi -skip_next=false -git log ${git_log_opts} | fmt -w 80 -p " " | while read -r ; do - if [[ ${skip_next} == "true" ]] ; then - skip_next=false - elif [[ ${REPLY} == " svn path="* ]] ; then - skip_next=true - else - echo "${REPLY}" - fi -done > "${RELEASE_DIR}/ChangeLog" || die "ChangeLog creation failed" - -cd "${RELEASE_BUILDDIR}" - -echo ">>> Creating release tarball ${RELEASE_TARBALL}" -tar --owner portage --group portage -cjf "${RELEASE_TARBALL}" "${RELEASE}" || \ - die "tarball creation failed" - -DISTDIR=$(portageq distdir) -if [[ -n ${DISTDIR} && -d ${DISTDIR} && -w ${DISTDIR} ]] ; then - echo ">>> Copying release tarball into ${DISTDIR}" - cp "${RELEASE_TARBALL}" "${DISTDIR}"/ || echo "!!! tarball copy failed" -fi - -if [[ -n ${UPLOAD_LOCATION} ]] ; then - echo ">>> Uploading ${RELEASE_TARBALL} to ${USER}dev.gentoo.org:${UPLOAD_LOCATION}" - scp "${RELEASE_TARBALL}" "${USER}dev.gentoo.org:${UPLOAD_LOCATION}" || die "upload failed" -else - du -h "${RELEASE_TARBALL}" -fi - -exit 0 diff --git a/pym/_emerge/Binpkg.py b/pym/_emerge/Binpkg.py index a740efdb9..ded6dfd4c 100644 --- a/pym/_emerge/Binpkg.py +++ b/pym/_emerge/Binpkg.py @@ -1,6 +1,7 @@ # Copyright 1999-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import _emerge.emergelog from _emerge.EbuildPhase import EbuildPhase from _emerge.BinpkgFetcher import BinpkgFetcher from _emerge.BinpkgEnvExtractor import BinpkgEnvExtractor @@ -87,9 +88,9 @@ class Binpkg(CompositeTask): waiting_msg = ("Fetching '%s' " + \ "in the background. " + \ - "To view fetch progress, run `tail -f %s" + \ - "/var/log/emerge-fetch.log` in another " + \ - "terminal.") % (prefetcher.pkg_path, settings["EPREFIX"]) + "To view fetch progress, run `tail -f %s` in another terminal.") \ + % (prefetcher.pkg_path, os.path.join( + _emerge.emergelog._emerge_log_dir, "emerge-fetch.log")) msg_prefix = colorize("GOOD", " * ") waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \ for line in textwrap.wrap(waiting_msg, 65)) @@ -344,7 +345,7 @@ class Binpkg(CompositeTask): def _chpathtool_exit(self, chpathtool): if self._final_exit(chpathtool) != os.EX_OK: self._unlock_builddir() - self._writemsg_level("!!! Error Adjusting Prefix to %s" % + self._writemsg_level("!!! Error Adjusting Prefix to %s\n" % (self.settings["EPREFIX"],), noiselevel=-1, level=logging.ERROR) self.wait() diff --git a/pym/_emerge/EbuildBuild.py b/pym/_emerge/EbuildBuild.py index f8dc305aa..1351c796b 100644 --- a/pym/_emerge/EbuildBuild.py +++ b/pym/_emerge/EbuildBuild.py @@ -1,6 +1,7 @@ # Copyright 1999-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import _emerge.emergelog from _emerge.EbuildExecuter import EbuildExecuter from _emerge.EbuildPhase import EbuildPhase from _emerge.EbuildBinpkg import EbuildBinpkg @@ -75,9 +76,8 @@ class EbuildBuild(CompositeTask): waiting_msg = "Fetching files " + \ "in the background. " + \ - "To view fetch progress, run `tail -f " + \ - EPREFIX + "/var/log/emerge-fetch.log` in another " + \ - "terminal." + "To view fetch progress, run `tail -f %s` in another terminal." \ + % (os.path.join(_emerge.emergelog._emerge_log_dir, "emerge-fetch.log")) msg_prefix = colorize("GOOD", " * ") from textwrap import wrap waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \ diff --git a/pym/_emerge/FakeVartree.py b/pym/_emerge/FakeVartree.py index 14be50c7f..254f667b7 100644 --- a/pym/_emerge/FakeVartree.py +++ b/pym/_emerge/FakeVartree.py @@ -299,7 +299,9 @@ def grab_global_updates(portdb): upd_commands.extend(commands) retupdates[repo_name] = upd_commands - master_repo = portdb.getRepositoryName(portdb.porttree_root) + master_repo = portdb.repositories.mainRepo() + if master_repo is not None: + master_repo = master_repo.name if master_repo in retupdates: retupdates['DEFAULT'] = retupdates[master_repo] diff --git a/pym/_emerge/MiscFunctionsProcess.py b/pym/_emerge/MiscFunctionsProcess.py index bada79d86..b7f5892a9 100644 --- a/pym/_emerge/MiscFunctionsProcess.py +++ b/pym/_emerge/MiscFunctionsProcess.py @@ -33,11 +33,15 @@ class MiscFunctionsProcess(AbstractEbuildProcess): if self._dummy_pipe_fd is not None: self.settings["PORTAGE_PIPE_FD"] = str(self._dummy_pipe_fd) + if "fakeroot" in self.settings.features: + kwargs["fakeroot"] = True + # Temporarily unset EBUILD_PHASE so that bashrc code doesn't # think this is a real phase. phase_backup = self.settings.pop("EBUILD_PHASE", None) try: - return spawn(" ".join(args), self.settings, **kwargs) + return spawn(" ".join(args), self.settings, + **portage._native_kwargs(kwargs)) finally: if phase_backup is not None: self.settings["EBUILD_PHASE"] = phase_backup diff --git a/pym/_emerge/PackageMerge.py b/pym/_emerge/PackageMerge.py index ef298ca48..fa2102fe1 100644 --- a/pym/_emerge/PackageMerge.py +++ b/pym/_emerge/PackageMerge.py @@ -1,7 +1,8 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.CompositeTask import CompositeTask +from portage.dep import _repo_separator from portage.output import colorize class PackageMerge(CompositeTask): __slots__ = ("merge",) @@ -29,7 +30,7 @@ class PackageMerge(CompositeTask): msg = "%s %s%s" % \ (action_desc, counter_str, - colorize(pkg_color, pkg.cpv)) + colorize(pkg_color, pkg.cpv + _repo_separator + pkg.repo)) if pkg.root_config.settings["ROOT"] != "/": msg += " %s %s" % (preposition, pkg.root) diff --git a/pym/_emerge/Scheduler.py b/pym/_emerge/Scheduler.py index dd268f708..d6db31108 100644 --- a/pym/_emerge/Scheduler.py +++ b/pym/_emerge/Scheduler.py @@ -1,7 +1,7 @@ # Copyright 1999-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function, unicode_literals +from __future__ import division, print_function, unicode_literals from collections import deque import gc diff --git a/pym/_emerge/UserQuery.py b/pym/_emerge/UserQuery.py new file mode 100644 index 000000000..c866a0d8c --- /dev/null +++ b/pym/_emerge/UserQuery.py @@ -0,0 +1,71 @@ +# Copyright 1999-2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import print_function + +import signal +import sys + +from portage.output import bold, create_color_func + + +class UserQuery(object): + """The UserQuery class is used to prompt the user with a set of responses, + as well as accepting and handling the responses.""" + + def __init__(self, myopts): + self.myopts = myopts + + def query(self, prompt, enter_invalid, responses=None, colours=None): + """Display a prompt and a set of responses, then waits for user input + and check it against the responses. The first match is returned. + + An empty response will match the first value in the list of responses, + unless enter_invalid is True. The input buffer is *not* cleared prior + to the prompt! + + prompt: The String to display as a prompt. + responses: a List of Strings with the acceptable responses. + colours: a List of Functions taking and returning a String, used to + process the responses for display. Typically these will be functions + like red() but could be e.g. lambda x: "DisplayString". + + If responses is omitted, it defaults to ["Yes", "No"], [green, red]. + If only colours is omitted, it defaults to [bold, ...]. + + Returns a member of the List responses. (If called without optional + arguments, it returns "Yes" or "No".) + + KeyboardInterrupt is converted to SystemExit to avoid tracebacks being + printed.""" + if responses is None: + responses = ["Yes", "No"] + colours = [ + create_color_func("PROMPT_CHOICE_DEFAULT"), + create_color_func("PROMPT_CHOICE_OTHER") + ] + elif colours is None: + colours=[bold] + colours=(colours*len(responses))[:len(responses)] + if "--alert" in self.myopts: + prompt = '\a' + prompt + print(bold(prompt), end=' ') + try: + while True: + if sys.hexversion >= 0x3000000: + response=input("["+"/".join([colours[i](responses[i]) + for i in range(len(responses))])+"] ") + else: + response=raw_input("["+"/".join([colours[i](responses[i]) + for i in range(len(responses))])+"] ") + if response or not enter_invalid: + for key in responses: + # An empty response will match the + # first value in responses. + if response.upper()==key[:len(response)].upper(): + return key + print("Sorry, response '%s' not understood." % response, + end=' ') + except (EOFError, KeyboardInterrupt): + print("Interrupted.") + sys.exit(128 + signal.SIGINT) diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py index 0777de88a..9f34f545c 100644 --- a/pym/_emerge/actions.py +++ b/pym/_emerge/actions.py @@ -1,7 +1,7 @@ # Copyright 1999-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function, unicode_literals +from __future__ import division, print_function, unicode_literals import errno import logging @@ -85,7 +85,7 @@ from _emerge.sync.old_tree_timestamp import old_tree_timestamp_warn from _emerge.unmerge import unmerge from _emerge.UnmergeDepPriority import UnmergeDepPriority from _emerge.UseFlagDisplay import pkg_use_display -from _emerge.userquery import userquery +from _emerge.UserQuery import UserQuery if sys.hexversion >= 0x3000000: long = int @@ -388,8 +388,9 @@ def action_build(settings, trees, mtimedb, else: prompt="Would you like to merge these packages?" print() + uq = UserQuery(myopts) if prompt is not None and "--ask" in myopts and \ - userquery(prompt, enter_invalid) == "No": + uq.query(prompt, enter_invalid) == "No": print() print("Quitting.") print() @@ -469,6 +470,7 @@ def action_build(settings, trees, mtimedb, def action_config(settings, trees, myopts, myfiles): enter_invalid = '--ask-enter-invalid' in myopts + uq = UserQuery(myopts) if len(myfiles) != 1: print(red("!!! config can only take a single package atom at this time\n")) sys.exit(1) @@ -498,7 +500,7 @@ def action_config(settings, trees, myopts, myfiles): print(options[-1]+") "+pkg) print("X) Cancel") options.append("X") - idx = userquery("Selection?", enter_invalid, responses=options) + idx = uq.query("Selection?", enter_invalid, responses=options) if idx == "X": sys.exit(128 + signal.SIGINT) pkg = pkgs[int(idx)-1] @@ -513,7 +515,7 @@ def action_config(settings, trees, myopts, myfiles): print() if "--ask" in myopts: - if userquery("Ready to configure %s?" % pkg, enter_invalid) == "No": + if uq.query("Ready to configure %s?" % pkg, enter_invalid) == "No": sys.exit(128 + signal.SIGINT) else: print("Configuring pkg...") @@ -558,10 +560,9 @@ def action_depclean(settings, trees, ldpath_mtimes, msg.append("\n") msg.append("As a safety measure, depclean will not remove any packages\n") msg.append("unless *all* required dependencies have been resolved. As a\n") - msg.append("consequence, it is often necessary to run %s\n" % \ - good("`emerge --update")) - msg.append(good("--newuse --deep @world`") + \ - " prior to depclean.\n") + msg.append("consequence of this, it often becomes necessary to run \n") + msg.append("%s" % good("`emerge --update --newuse --deep @world`") + + " prior to depclean.\n") if action == "depclean" and "--quiet" not in myopts and not myfiles: portage.writemsg_stdout("\n") @@ -1366,7 +1367,8 @@ def action_deselect(settings, trees, opts, atoms): if '--ask' in opts: prompt = "Would you like to remove these " + \ "packages from your world favorites?" - if userquery(prompt, enter_invalid) == 'No': + uq = UserQuery(opts) + if uq.query(prompt, enter_invalid) == 'No': return 128 + signal.SIGINT remaining = set(world_set) @@ -1498,14 +1500,14 @@ def action_info(settings, trees, myopts, myfiles): vm_info = get_vm_info() if "ram.total" in vm_info: - line = "%-9s %10d total" % ("KiB Mem:", vm_info["ram.total"] / 1024) + line = "%-9s %10d total" % ("KiB Mem:", vm_info["ram.total"] // 1024) if "ram.free" in vm_info: - line += ",%10d free" % (vm_info["ram.free"] / 1024,) + line += ",%10d free" % (vm_info["ram.free"] // 1024,) append(line) if "swap.total" in vm_info: - line = "%-9s %10d total" % ("KiB Swap:", vm_info["swap.total"] / 1024) + line = "%-9s %10d total" % ("KiB Swap:", vm_info["swap.total"] // 1024) if "swap.free" in vm_info: - line += ",%10d free" % (vm_info["swap.free"] / 1024,) + line += ",%10d free" % (vm_info["swap.free"] // 1024,) append(line) lastSync = portage.grabfile(os.path.join( @@ -1581,7 +1583,9 @@ def action_info(settings, trees, myopts, myfiles): myvars = sorted(set(atoms)) - main_repo = portdb.getRepositoryName(portdb.porttree_root) + main_repo = portdb.repositories.mainRepo() + if main_repo is not None: + main_repo = main_repo.name cp_map = {} cp_max_len = 0 @@ -2344,7 +2348,8 @@ def _sync_repo(emerge_config, repo): family, socket.SOCK_STREAM)) except socket.error as e: writemsg_level( - "!!! getaddrinfo failed for '%s': %s\n" % (hostname, e), + "!!! getaddrinfo failed for '%s': %s\n" % (hostname, + _unicode_decode(e.strerror, encoding=_encodings['stdio'])), noiselevel=-1, level=logging.ERROR) if addrinfos: @@ -2404,7 +2409,8 @@ def _sync_repo(emerge_config, repo): if (retries==0): if "--ask" in myopts: - if userquery("Do you want to sync your Portage tree " + \ + uq = UserQuery(myopts) + if uq.query("Do you want to sync your Portage tree " + \ "with the mirror at\n" + blue(dosyncuri) + bold("?"), enter_invalid) == "No": print() @@ -3843,7 +3849,8 @@ def run_action(emerge_config): (access_desc,), noiselevel=-1) if portage.data.secpass < 1 and not need_superuser: portage.data.portage_group_warning() - if userquery("Would you like to add --pretend to options?", + uq = UserQuery(emerge_config.opts) + if uq.query("Would you like to add --pretend to options?", "--ask-enter-invalid" in emerge_config.opts) == "No": return 128 + signal.SIGINT emerge_config.opts["--pretend"] = True @@ -4055,8 +4062,21 @@ def run_action(emerge_config): # GLEP 42 says to display news *after* an emerge --pretend if "--pretend" not in emerge_config.opts: - display_news_notification( - emerge_config.target_config, emerge_config.opts) + uq = UserQuery(emerge_config.opts) + if display_news_notification(emerge_config.target_config, + emerge_config.opts) \ + and "--ask" in emerge_config.opts \ + and uq.query("Would you like to read the news items while " \ + "calculating dependencies?", + '--ask-enter-invalid' in emerge_config.opts) == "Yes": + try: + subprocess.call(['eselect', 'news', 'read']) + # If eselect is not installed, Python <3.3 will throw an + # OSError. >=3.3 will throw a FileNotFoundError, which is a + # subclass of OSError. + except OSError: + writemsg("Please install eselect to use this feature.\n", + noiselevel=-1) retval = action_build(emerge_config.target_config.settings, emerge_config.trees, emerge_config.target_config.mtimedb, emerge_config.opts, emerge_config.action, diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py index c17f933f7..2f315bb0a 100644 --- a/pym/_emerge/depgraph.py +++ b/pym/_emerge/depgraph.py @@ -1,7 +1,7 @@ # Copyright 1999-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import print_function, unicode_literals +from __future__ import division, print_function, unicode_literals import collections import errno @@ -73,7 +73,7 @@ from _emerge.SetArg import SetArg from _emerge.show_invalid_depstring_notice import show_invalid_depstring_notice from _emerge.UnmergeDepPriority import UnmergeDepPriority from _emerge.UseFlagDisplay import pkg_use_display -from _emerge.userquery import userquery +from _emerge.UserQuery import UserQuery from _emerge.resolver.backtracking import Backtracker, BacktrackParameter from _emerge.resolver.package_tracker import PackageTracker, PackageTrackerDbapiWrapper @@ -107,7 +107,7 @@ def _wildcard_set(atoms): class _frozen_depgraph_config(object): - def __init__(self, settings, trees, myopts, spinner): + def __init__(self, settings, trees, myopts, params, spinner): self.settings = settings self.target_root = settings["EROOT"] self.myopts = myopts @@ -115,6 +115,7 @@ class _frozen_depgraph_config(object): if settings.get("PORTAGE_DEBUG", "") == "1": self.edebug = 1 self.spinner = spinner + self.requested_depth = params.get("deep", 0) self._running_root = trees[trees._running_eroot]["root_config"] self.pkgsettings = {} self.trees = {} @@ -420,6 +421,7 @@ class _dynamic_depgraph_config(object): self._buildpkgonly_deps_unsatisfied = False self._autounmask = depgraph._frozen_config.myopts.get('--autounmask') != 'n' self._success_without_autounmask = False + self._required_use_unsatisfied = False self._traverse_ignored_deps = False self._complete_mode = False self._slot_operator_deps = {} @@ -502,13 +504,18 @@ class _dynamic_depgraph_config(object): class depgraph(object): + # Represents the depth of a node that is unreachable from explicit + # user arguments (or their deep dependencies). Such nodes are pulled + # in by the _complete_graph method. + _UNREACHABLE_DEPTH = object() + pkg_tree_map = RootConfig.pkg_tree_map def __init__(self, settings, trees, myopts, myparams, spinner, frozen_config=None, backtrack_parameters=BacktrackParameter(), allow_backtracking=False): if frozen_config is None: frozen_config = _frozen_depgraph_config(settings, trees, - myopts, spinner) + myopts, myparams, spinner) self._frozen_config = frozen_config self._dynamic_config = _dynamic_depgraph_config(self, myparams, allow_backtracking, backtrack_parameters) @@ -520,6 +527,8 @@ class depgraph(object): self._event_loop = (portage._internal_caller and global_event_loop() or EventLoop(main=False)) + self.query = UserQuery(myopts).query + def _load_vdb(self): """ Load installed package metadata if appropriate. This used to be called @@ -639,23 +648,96 @@ class depgraph(object): # Go through all slot operator deps and check if one of these deps # has a parent that is matched by one of the atoms from above. forced_rebuilds = {} - for (root, slot_atom), deps in self._dynamic_config._slot_operator_deps.items(): - rebuild_atoms = atoms.get(root, set()) - for dep in deps: - if getattr(dep.parent, "installed", False) or dep.child.installed or \ - dep.parent.slot_atom not in rebuild_atoms: - continue + for root, rebuild_atoms in atoms.items(): - # Make sure the child's slot/subslot has changed. If it hasn't, - # then another child has forced this rebuild. - installed_pkg = self._select_pkg_from_installed(root, dep.child.slot_atom)[0] - if installed_pkg and installed_pkg.slot == dep.child.slot and \ - installed_pkg.sub_slot == dep.child.sub_slot: + for slot_atom in rebuild_atoms: + + inst_pkg, reinst_pkg = \ + self._select_pkg_from_installed(root, slot_atom) + + if inst_pkg is reinst_pkg or reinst_pkg is None: continue - # The child has forced a rebuild of the parent - forced_rebuilds.setdefault(root, {}).setdefault(dep.child, set()).add(dep.parent) + # Generate pseudo-deps for any slot-operator deps of + # inst_pkg. Its deps aren't in _slot_operator_deps + # because it hasn't been added to the graph, but we + # are interested in any rebuilds that it triggered. + built_slot_op_atoms = [] + if inst_pkg is not None: + selected_atoms = self._select_atoms_probe( + inst_pkg.root, inst_pkg) + for atom in selected_atoms: + if atom.slot_operator_built: + built_slot_op_atoms.append(atom) + + if not built_slot_op_atoms: + continue + + # Use a cloned list, since we may append to it below. + deps = self._dynamic_config._slot_operator_deps.get( + (root, slot_atom), [])[:] + + if built_slot_op_atoms and reinst_pkg is not None: + for child in self._dynamic_config.digraph.child_nodes( + reinst_pkg): + + if child.installed: + continue + + for atom in built_slot_op_atoms: + # NOTE: Since atom comes from inst_pkg, and + # reinst_pkg is the replacement parent, there's + # no guarantee that atom will completely match + # child. So, simply use atom.cp and atom.slot + # for matching. + if atom.cp != child.cp: + continue + if atom.slot and atom.slot != child.slot: + continue + deps.append(Dependency(atom=atom, child=child, + root=child.root, parent=reinst_pkg)) + + for dep in deps: + if dep.child.installed: + # Find the replacement child. + child = next((pkg for pkg in + self._dynamic_config._package_tracker.match( + dep.root, dep.child.slot_atom) + if not pkg.installed), None) + + if child is None: + continue + + inst_child = dep.child.installed + + else: + child = dep.child + inst_child = self._select_pkg_from_installed( + child.root, child.slot_atom)[0] + + # Make sure the child's slot/subslot has changed. If it + # hasn't, then another child has forced this rebuild. + if inst_child and inst_child.slot == child.slot and \ + inst_child.sub_slot == child.sub_slot: + continue + + if dep.parent.installed: + # Find the replacement parent. + parent = next((pkg for pkg in + self._dynamic_config._package_tracker.match( + dep.parent.root, dep.parent.slot_atom) + if not pkg.installed), None) + + if parent is None: + continue + + else: + parent = dep.parent + + # The child has forced a rebuild of the parent + forced_rebuilds.setdefault(root, {} + ).setdefault(child, set()).add(parent) if debug: writemsg_level("slot operator dependencies:\n", @@ -990,6 +1072,11 @@ class depgraph(object): if not conflicts: return + if debug: + writemsg_level( + "\n!!! Slot conflict handler started.\n", + level=logging.DEBUG, noiselevel=-1) + # Get a set of all conflicting packages. conflict_pkgs = set() for conflict in conflicts: @@ -1043,7 +1130,15 @@ class depgraph(object): def __str__(self): return "(%s)" % ",".join(str(pkg) for pkg in self) + non_matching_forced = set() for conflict in conflicts: + if debug: + writemsg_level(" conflict:\n", level=logging.DEBUG, noiselevel=-1) + writemsg_level(" root: %s\n" % conflict.root, level=logging.DEBUG, noiselevel=-1) + writemsg_level(" atom: %s\n" % conflict.atom, level=logging.DEBUG, noiselevel=-1) + for pkg in conflict: + writemsg_level(" pkg: %s\n" % pkg, level=logging.DEBUG, noiselevel=-1) + all_parent_atoms = set() for pkg in conflict: all_parent_atoms.update( @@ -1051,9 +1146,16 @@ class depgraph(object): for parent, atom in all_parent_atoms: is_arg_parent = isinstance(parent, AtomArg) + is_non_conflict_parent = parent not in conflict_pkgs and \ + parent not in indirect_conflict_pkgs - if parent not in conflict_pkgs and \ - parent not in indirect_conflict_pkgs: + if debug: + writemsg_level(" parent: %s\n" % parent, level=logging.DEBUG, noiselevel=-1) + writemsg_level(" arg, non-conflict: %s, %s\n" % (is_arg_parent, is_non_conflict_parent), + level=logging.DEBUG, noiselevel=-1) + writemsg_level(" atom: %s\n" % atom, level=logging.DEBUG, noiselevel=-1) + + if is_non_conflict_parent: parent = non_conflict_node atom_set = InternalPackageSet( @@ -1065,14 +1167,31 @@ class depgraph(object): modified_use=self._pkg_use_enabled(pkg)) and \ not (is_arg_parent and pkg.installed): matched.append(pkg) - if len(matched) == len(conflict): - # All packages match. - continue + + if debug: + for match in matched: + writemsg_level(" match: %s\n" % match, level=logging.DEBUG, noiselevel=-1) + + if len(matched) > 1: + # Even if all packages match, this parent must still + # be added to the conflict_graph. Otherwise, we risk + # removing all of these packages from the depgraph, + # which could cause a missed update (bug #522084). + conflict_graph.add(or_tuple(matched), parent) elif len(matched) == 1: conflict_graph.add(matched[0], parent) else: - # More than one packages matched, but not all. - conflict_graph.add(or_tuple(matched), parent) + # This typically means that autounmask broke a + # USE-dep, but it could also be due to the slot + # not matching due to multislot (bug #220341). + # Either way, don't try to solve this conflict. + # Instead, force them all into the graph so that + # they are protected from removal. + non_matching_forced.update(conflict) + if debug: + for pkg in conflict: + writemsg_level(" non-match: %s\n" % pkg, + level=logging.DEBUG, noiselevel=-1) for pkg in indirect_conflict_pkgs: for parent, atom in self._dynamic_config._parent_atoms.get(pkg, []): @@ -1090,6 +1209,7 @@ class depgraph(object): # Now select required packages. Collect them in the # 'forced' set. forced = set([non_conflict_node]) + forced.update(non_matching_forced) unexplored = set([non_conflict_node]) # or_tuples get special handling. We first explore # all packages in the hope of having forced one of @@ -1336,6 +1456,10 @@ class depgraph(object): found_update = False for parent_atom, conflict_pkgs in conflict_atoms.items(): parent, atom = parent_atom + + if not isinstance(parent, Package): + continue + if atom.slot_operator != "=" or not parent.built: continue @@ -1534,14 +1658,40 @@ class depgraph(object): selective = "selective" in self._dynamic_config.myparams want_downgrade = None - def check_reverse_dependencies(existing_pkg, candidate_pkg): + def check_reverse_dependencies(existing_pkg, candidate_pkg, + replacement_parent=None): """ Check if candidate_pkg satisfies all of existing_pkg's non- slot operator parents. """ + built_slot_operator_parents = set() for parent, atom in self._dynamic_config._parent_atoms.get(existing_pkg, []): - if atom.slot_operator == "=" and parent.built: - continue + if atom.slot_operator_built: + built_slot_operator_parents.add(parent) + + for parent, atom in self._dynamic_config._parent_atoms.get(existing_pkg, []): + if isinstance(parent, Package): + if parent in built_slot_operator_parents: + # This parent may need to be rebuilt, so its + # dependencies aren't necessarily relevant. + continue + + if replacement_parent is not None and \ + (replacement_parent.slot_atom == parent.slot_atom + or replacement_parent.cpv == parent.cpv): + # This parent is irrelevant because we intend to + # replace it with replacement_parent. + continue + + if any(pkg is not parent and + (pkg.slot_atom == parent.slot_atom or + pkg.cpv == parent.cpv) for pkg in + self._dynamic_config._package_tracker.match( + parent.root, Atom(parent.cp))): + # This parent may need to be eliminated due to a + # slot conflict, so its dependencies aren't + # necessarily relevant. + continue atom_set = InternalPackageSet(initial_atoms=(atom,), allow_repo=True) @@ -1660,7 +1810,8 @@ class depgraph(object): continue if not insignificant and \ - check_reverse_dependencies(dep.child, pkg): + check_reverse_dependencies(dep.child, pkg, + replacement_parent=replacement_parent): candidate_pkg_atoms.append((pkg, unevaluated_atom)) candidate_pkgs.append(pkg) @@ -2035,6 +2186,13 @@ class depgraph(object): arg = arg_stack.pop() if arg in traversed_set_args: continue + + # If a node with the same hash already exists in + # the digraph, preserve the existing instance which + # may have a different reset_depth attribute + # (distiguishes user arguments from sets added for + # another reason such as complete mode). + arg = self._dynamic_config.digraph.get(arg, arg) traversed_set_args.add(arg) if add_to_digraph: @@ -2054,8 +2212,16 @@ class depgraph(object): if nested_set is None: nested_set = root_config.sets.get(s) if nested_set is not None: + # Propagate the reset_depth attribute from + # parent set to nested set. nested_arg = SetArg(arg=token, pset=nested_set, + reset_depth=arg.reset_depth, root_config=root_config) + + # Preserve instances already in the graph (same + # reason as for the "arg" variable above). + nested_arg = self._dynamic_config.digraph.get( + nested_arg, nested_arg) arg_stack.append(nested_arg) if add_to_digraph: self._dynamic_config.digraph.add(nested_arg, arg, @@ -2104,9 +2270,42 @@ class depgraph(object): dep.collapsed_priority.ignored): # This is an unnecessary build-time dep. return 1 + + # NOTE: For removal actions, allow_unsatisfied is always + # True since all existing removal actions traverse all + # installed deps deeply via the _complete_graph method, + # which calls _create_graph with allow_unsatisfied = True. if allow_unsatisfied: self._dynamic_config._unsatisfied_deps.append(dep) return 1 + + # The following case occurs when + # _solve_non_slot_operator_slot_conflicts calls + # _create_graph. In this case, ignore unsatisfied deps for + # installed packages only if their depth is beyond the depth + # requested by the user and the dep was initially + # unsatisfied (not broken by a slot conflict in the current + # graph). See bug #520950. + # NOTE: The value of dep.parent.depth is guaranteed to be + # either an integer or _UNREACHABLE_DEPTH, where + # _UNREACHABLE_DEPTH indicates that the parent has been + # pulled in by the _complete_graph method (rather than by + # explicit arguments or their deep dependencies). These + # cases must be distinguished because depth is meaningless + # for packages that are not reachable as deep dependencies + # of arguments. + if (self._dynamic_config._complete_mode and + isinstance(dep.parent, Package) and + dep.parent.installed and + (dep.parent.depth is self._UNREACHABLE_DEPTH or + (self._frozen_config.requested_depth is not True and + dep.parent.depth >= self._frozen_config.requested_depth))): + inst_pkg, in_graph = \ + self._select_pkg_from_installed(dep.root, dep.atom) + if inst_pkg is None: + self._dynamic_config._initially_unsatisfied_deps.append(dep) + return 1 + self._dynamic_config._unsatisfied_deps_for_display.append( ((dep.root, dep.atom), {"myparent":dep.parent})) @@ -2263,6 +2462,7 @@ class depgraph(object): self._dynamic_config._unsatisfied_deps_for_display.append( ((pkg.root, atom), {"myparent" : dep.parent, "show_req_use" : pkg})) + self._dynamic_config._required_use_unsatisfied = True self._dynamic_config._skip_restart = True return 0 @@ -2351,14 +2551,23 @@ class depgraph(object): # Installing package A, we need to make sure package A's deps are met. # emerge --deep <pkgspec>; we need to recursively check dependencies of pkgspec # If we are in --nodeps (no recursion) mode, we obviously only check 1 level of dependencies. - if arg_atoms and depth > 0: + if arg_atoms and depth != 0: for parent, atom in arg_atoms: if parent.reset_depth: depth = 0 break - if previously_added and pkg.depth is not None: - depth = min(pkg.depth, depth) + if previously_added and depth != 0 and \ + isinstance(pkg.depth, int): + # Use pkg.depth if it is less than depth. + if isinstance(depth, int): + depth = min(pkg.depth, depth) + else: + # depth is _UNREACHABLE_DEPTH and pkg.depth is + # an int, so use the int because it's considered + # to be less than _UNREACHABLE_DEPTH. + depth = pkg.depth + pkg.depth = depth deep = self._dynamic_config.myparams.get("deep", 0) update = "--update" in self._frozen_config.myopts @@ -2656,7 +2865,11 @@ class depgraph(object): def _wrapped_add_pkg_dep_string(self, pkg, dep_root, dep_priority, dep_string, allow_unsatisfied): - depth = pkg.depth + 1 + if isinstance(pkg.depth, int): + depth = pkg.depth + 1 + else: + depth = pkg.depth + deep = self._dynamic_config.myparams.get("deep", 0) recurse_satisfied = deep is True or depth <= deep debug = "--debug" in self._frozen_config.myopts @@ -3484,9 +3697,9 @@ class depgraph(object): if not self._add_pkg(arg.package, dep) or \ not self._create_graph(): if not self.need_restart(): - sys.stderr.write(("\n\n!!! Problem " + \ + writemsg(("\n\n!!! Problem " + \ "resolving dependencies for %s\n") % \ - arg.arg) + arg.arg, noiselevel=-1) return 0, myfavorites continue if debug: @@ -3887,10 +4100,14 @@ class depgraph(object): # Recursively traversed virtual dependencies, and their # direct dependencies, are considered to have the same # depth as direct dependencies. - if parent.depth is None: - virt_depth = None - else: + if isinstance(parent.depth, int): virt_depth = parent.depth + 1 + else: + # The depth may be None when called via + # _select_atoms_probe, or it may be + # _UNREACHABLE_DEPTH for complete mode. + virt_depth = parent.depth + chosen_atom_ids = frozenset(id(atom) for atom in mycheck[1]) selected_atoms = OrderedDict() node_stack = [(parent, None, None)] @@ -4776,7 +4993,8 @@ class depgraph(object): raise depth = parent.depth or 0 - depth += 1 + if isinstance(depth, int): + depth += 1 if arg_atoms: for arg, atom in arg_atoms: @@ -5773,14 +5991,14 @@ class depgraph(object): pset = root_config.sets[s] atom = SETPREFIX + s args.append(SetArg(arg=atom, pset=pset, - root_config=root_config)) + reset_depth=False, root_config=root_config)) self._set_args(args) for arg in self._expand_set_args(args, add_to_digraph=True): for atom in arg.pset.getAtoms(): self._dynamic_config._dep_stack.append( Dependency(atom=atom, root=arg.root_config.root, - parent=arg)) + parent=arg, depth=self._UNREACHABLE_DEPTH)) if True: if self._dynamic_config._ignored_deps: @@ -7299,12 +7517,14 @@ class depgraph(object): (using CONFIG_PROTECT). The message includes the comments and the changes. """ - autounmask_write = self._frozen_config.myopts.get("--autounmask-write", "n") == True + ask = "--ask" in self._frozen_config.myopts + autounmask_write = \ + self._frozen_config.myopts.get("--autounmask-write", + ask) is True autounmask_unrestricted_atoms = \ self._frozen_config.myopts.get("--autounmask-unrestricted-atoms", "n") == True quiet = "--quiet" in self._frozen_config.myopts pretend = "--pretend" in self._frozen_config.myopts - ask = "--ask" in self._frozen_config.myopts enter_invalid = '--ask-enter-invalid' in self._frozen_config.myopts def check_if_latest(pkg): @@ -7624,7 +7844,7 @@ class depgraph(object): if ask and write_to_file and file_to_write_to: prompt = "\nWould you like to add these " + \ "changes to your config files?" - if userquery(prompt, enter_invalid) == 'No': + if self.query(prompt, enter_invalid) == 'No': write_to_file = False if write_to_file and file_to_write_to: @@ -7726,18 +7946,23 @@ class depgraph(object): break if world_problems: - sys.stderr.write("\n!!! Problems have been " + \ - "detected with your world file\n") - sys.stderr.write("!!! Please run " + \ - green("emaint --check world")+"\n\n") + writemsg("\n!!! Problems have been " + \ + "detected with your world file\n", + noiselevel=-1) + writemsg("!!! Please run " + \ + green("emaint --check world")+"\n\n", + noiselevel=-1) if self._dynamic_config._missing_args: - sys.stderr.write("\n" + colorize("BAD", "!!!") + \ - " Ebuilds for the following packages are either all\n") - sys.stderr.write(colorize("BAD", "!!!") + \ - " masked or don't exist:\n") - sys.stderr.write(" ".join(str(atom) for arg, atom in \ - self._dynamic_config._missing_args) + "\n") + writemsg("\n" + colorize("BAD", "!!!") + \ + " Ebuilds for the following packages are either all\n", + noiselevel=-1) + writemsg(colorize("BAD", "!!!") + \ + " masked or don't exist:\n", + noiselevel=-1) + writemsg(" ".join(str(atom) for arg, atom in \ + self._dynamic_config._missing_args) + "\n", + noiselevel=-1) if self._dynamic_config._pprovided_args: arg_refs = {} @@ -7777,7 +8002,7 @@ class depgraph(object): msg.append(" C) Remove offending entries from package.provided.\n\n") msg.append("The best course of action depends on the reason that an offending\n") msg.append("package.provided entry exists.\n\n") - sys.stderr.write("".join(msg)) + writemsg("".join(msg), noiselevel=-1) masked_packages = [] for pkg in self._dynamic_config._masked_license_updates: @@ -7891,7 +8116,7 @@ class depgraph(object): "favorites?" enter_invalid = '--ask-enter-invalid' in \ self._frozen_config.myopts - if userquery(prompt, enter_invalid) == "No": + if self.query(prompt, enter_invalid) == "No": skip = True if not skip: @@ -8170,8 +8395,9 @@ class depgraph(object): return self._dynamic_config._need_restart and \ not self._dynamic_config._skip_restart - def success_without_autounmask(self): - return self._dynamic_config._success_without_autounmask + def need_config_change(self): + return self._dynamic_config._success_without_autounmask or \ + self._dynamic_config._required_use_unsatisfied def autounmask_breakage_detected(self): try: @@ -8226,18 +8452,20 @@ class _dep_check_composite_db(dbapi): return ret - def match(self, atom): + def match_pkgs(self, atom): cache_key = (atom, atom.unevaluated_atom) ret = self._match_cache.get(cache_key) if ret is not None: + for pkg in ret: + self._cpv_pkg_map[pkg.cpv] = pkg return ret[:] + atom_set = InternalPackageSet(initial_atoms=(atom,)) ret = [] pkg, existing = self._depgraph._select_package(self._root, atom) - if pkg is not None and self._visible(pkg): - self._cpv_pkg_map[pkg.cpv] = pkg - ret.append(pkg.cpv) + if pkg is not None and self._visible(pkg, atom_set): + ret.append(pkg) if pkg is not None and \ atom.slot is None and \ @@ -8268,18 +8496,19 @@ class _dep_check_composite_db(dbapi): self._root, slot_atom) if not pkg: continue - if not self._visible(pkg): + if not self._visible(pkg, atom_set): continue - self._cpv_pkg_map[pkg.cpv] = pkg - ret.append(pkg.cpv) + ret.append(pkg) if len(ret) > 1: - self._cpv_sort_ascending(ret) + ret.sort() self._match_cache[cache_key] = ret + for pkg in ret: + self._cpv_pkg_map[pkg.cpv] = pkg return ret[:] - def _visible(self, pkg): + def _visible(self, pkg, atom_set): if pkg.installed and not self._depgraph._want_installed_pkg(pkg): return False if pkg.installed and \ @@ -8317,6 +8546,11 @@ class _dep_check_composite_db(dbapi): elif in_graph != pkg: # Mask choices for packages that would trigger a slot # conflict with a previously selected package. + if not atom_set.findAtomForPackage(in_graph, + modified_use=self._depgraph._pkg_use_enabled(in_graph)): + # Only mask if the graph package matches the given + # atom (fixes bug #515230). + return True return False return True @@ -8324,8 +8558,8 @@ class _dep_check_composite_db(dbapi): metadata = self._cpv_pkg_map[cpv]._metadata return [metadata.get(x, "") for x in wants] - def match_pkgs(self, atom): - return [self._cpv_pkg_map[cpv] for cpv in self.match(atom)] + def match(self, atom): + return [pkg.cpv for pkg in self.match_pkgs(atom)] def ambiguous_package_name(arg, atoms, root_config, spinner, myopts): @@ -8413,13 +8647,13 @@ def _backtrack_depgraph(settings, trees, myopts, myparams, myaction, myfiles, sp debug = "--debug" in myopts mydepgraph = None max_retries = myopts.get('--backtrack', 10) - max_depth = max(1, (max_retries + 1) / 2) + max_depth = max(1, (max_retries + 1) // 2) allow_backtracking = max_retries > 0 backtracker = Backtracker(max_depth) backtracked = 0 frozen_config = _frozen_depgraph_config(settings, trees, - myopts, spinner) + myopts, myparams, spinner) while backtracker: @@ -8437,7 +8671,7 @@ def _backtrack_depgraph(settings, trees, myopts, myparams, myaction, myfiles, sp backtrack_parameters=backtrack_parameters) success, favorites = mydepgraph.select_files(myfiles) - if success or mydepgraph.success_without_autounmask(): + if success or mydepgraph.need_config_change(): break elif not allow_backtracking: break @@ -8449,7 +8683,7 @@ def _backtrack_depgraph(settings, trees, myopts, myparams, myaction, myfiles, sp else: break - if not (success or mydepgraph.success_without_autounmask()) and backtracked: + if not (success or mydepgraph.need_config_change()) and backtracked: if debug: writemsg_level( @@ -8501,7 +8735,7 @@ def _resume_depgraph(settings, trees, mtimedb, myopts, myparams, spinner): mergelist = mtimedb["resume"]["mergelist"] dropped_tasks = {} frozen_config = _frozen_depgraph_config(settings, trees, - myopts, spinner) + myopts, myparams, spinner) while True: mydepgraph = depgraph(settings, trees, myopts, myparams, spinner, frozen_config=frozen_config) diff --git a/pym/_emerge/main.py b/pym/_emerge/main.py index 4f0166eca..e2ddf8700 100644 --- a/pym/_emerge/main.py +++ b/pym/_emerge/main.py @@ -125,6 +125,7 @@ def insert_optional_args(args): new_args = [] default_arg_opts = { + '--alert' : y_or_n, '--ask' : y_or_n, '--autounmask' : y_or_n, '--autounmask-keep-masks': y_or_n, @@ -170,6 +171,7 @@ def insert_optional_args(args): # since existence of -n makes it too ambiguous. short_arg_opts_n = { 'a' : y_or_n, + 'A' : y_or_n, 'b' : y_or_n, 'g' : y_or_n, 'G' : y_or_n, @@ -301,6 +303,12 @@ def parse_opts(tmpcmdline, silent=False): true_y = ("True", "y") argument_options = { + "--alert": { + "shortopt" : "-A", + "help" : "alert (terminal bell) on prompts", + "choices" : true_y_or_n + }, + "--ask": { "shortopt" : "-a", "help" : "prompt before performing any actions", @@ -676,6 +684,11 @@ def parse_opts(tmpcmdline, silent=False): myoptions, myargs = parser.parse_known_args(args=tmpcmdline) + if myoptions.alert in true_y: + myoptions.alert = True + else: + myoptions.alert = None + if myoptions.ask in true_y: myoptions.ask = True else: @@ -987,7 +1000,10 @@ def emerge_main(args=None): args = portage._decode_argv(args) # Use system locale. - locale.setlocale(locale.LC_ALL, '') + try: + locale.setlocale(locale.LC_ALL, '') + except locale.Error as e: + writemsg_level("setlocale: %s\n" % e, level=logging.WARN) # Disable color until we're sure that it should be enabled (after # EMERGE_DEFAULT_OPTS has been parsed). diff --git a/pym/_emerge/post_emerge.py b/pym/_emerge/post_emerge.py index d5f1ba5fa..0cb533cf8 100644 --- a/pym/_emerge/post_emerge.py +++ b/pym/_emerge/post_emerge.py @@ -37,11 +37,14 @@ def clean_logs(settings): def display_news_notification(root_config, myopts): if "news" not in root_config.settings.features: - return + return False portdb = root_config.trees["porttree"].dbapi vardb = root_config.trees["vartree"].dbapi news_counts = count_unread_news(portdb, vardb) + if all(v == 0 for v in news_counts.values()): + return False display_news_notifications(news_counts) + return True def show_depclean_suggestion(): out = portage.output.EOutput() diff --git a/pym/_emerge/resolver/output_helpers.py b/pym/_emerge/resolver/output_helpers.py index e0ee3e099..eb8d97db8 100644 --- a/pym/_emerge/resolver/output_helpers.py +++ b/pym/_emerge/resolver/output_helpers.py @@ -53,7 +53,7 @@ class _RepoDisplay(object): # pre-allocate index for PORTDIR so that it always has index 0. for root_config in roots.values(): portdb = root_config.trees["porttree"].dbapi - portdir = portdb.porttree_root + portdir = portdb.repositories.mainRepoLocation() if portdir: self.repoStr(portdir) diff --git a/pym/_emerge/resolver/package_tracker.py b/pym/_emerge/resolver/package_tracker.py index 5982750a0..406d5cee2 100644 --- a/pym/_emerge/resolver/package_tracker.py +++ b/pym/_emerge/resolver/package_tracker.py @@ -152,7 +152,7 @@ class PackageTracker(object): packages may also be returned. """ cp_key = root, atom.cp - cache_key = root, atom, installed + cache_key = root, atom, atom.unevaluated_atom, installed try: return iter(self._match_cache.get(cp_key, {})[cache_key]) except KeyError: diff --git a/pym/_emerge/sync/old_tree_timestamp.py b/pym/_emerge/sync/old_tree_timestamp.py index 9b35aedd6..aa23a273e 100644 --- a/pym/_emerge/sync/old_tree_timestamp.py +++ b/pym/_emerge/sync/old_tree_timestamp.py @@ -1,6 +1,8 @@ -# Copyright 2010 Gentoo Foundation +# Copyright 2010-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import division + import locale import logging import time @@ -27,16 +29,16 @@ def whenago(seconds): out = [] if sec > 60: - mins = sec / 60 + mins = sec // 60 sec = sec % 60 if mins > 60: - hrs = mins / 60 + hrs = mins // 60 mins = mins % 60 if hrs > 24: - days = hrs / 24 + days = hrs // 24 hrs = hrs % 24 if days > 365: - years = days / 365 + years = days // 365 days = days % 365 if years: diff --git a/pym/_emerge/unmerge.py b/pym/_emerge/unmerge.py index b04f8f376..30b1930e9 100644 --- a/pym/_emerge/unmerge.py +++ b/pym/_emerge/unmerge.py @@ -17,8 +17,8 @@ from portage.versions import cpv_sort_key, _pkg_str from _emerge.emergelog import emergelog from _emerge.Package import Package +from _emerge.UserQuery import UserQuery from _emerge.UninstallFailure import UninstallFailure -from _emerge.userquery import userquery from _emerge.countdown import countdown def _unmerge_display(root_config, myopts, unmerge_action, @@ -488,7 +488,8 @@ def _unmerge_display(root_config, myopts, unmerge_action, if quiet: writemsg_level("\n", noiselevel=-1) - writemsg_level("\nAll selected packages: %s\n" % " ".join(all_selected), noiselevel=-1) + writemsg_level("\nAll selected packages: %s\n" % + " ".join('=%s' % x for x in all_selected), noiselevel=-1) writemsg_level("\n>>> " + colorize("UNMERGE_WARN", "'Selected'") + \ " packages are slated for removal.\n") @@ -529,7 +530,8 @@ def unmerge(root_config, myopts, unmerge_action, #we're done... return return os.EX_OK if "--ask" in myopts: - if userquery("Would you like to unmerge these packages?", + uq = UserQuery(myopts) + if uq.query("Would you like to unmerge these packages?", enter_invalid) == "No": # enter pretend mode for correct formatting of results myopts["--pretend"] = True diff --git a/pym/_emerge/userquery.py b/pym/_emerge/userquery.py deleted file mode 100644 index efae80aa6..000000000 --- a/pym/_emerge/userquery.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 1999-2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from __future__ import print_function - -import signal -import sys - -from portage.output import bold, create_color_func - -def userquery(prompt, enter_invalid, responses=None, colours=None): - """Displays a prompt and a set of responses, then waits for a response - which is checked against the responses and the first to match is - returned. An empty response will match the first value in responses, - unless enter_invalid is True. The input buffer is *not* cleared prior - to the prompt! - - prompt: a String. - responses: a List of Strings. - colours: a List of Functions taking and returning a String, used to - process the responses for display. Typically these will be functions - like red() but could be e.g. lambda x: "DisplayString". - If responses is omitted, defaults to ["Yes", "No"], [green, red]. - If only colours is omitted, defaults to [bold, ...]. - - Returns a member of the List responses. (If called without optional - arguments, returns "Yes" or "No".) - KeyboardInterrupt is converted to SystemExit to avoid tracebacks being - printed.""" - if responses is None: - responses = ["Yes", "No"] - colours = [ - create_color_func("PROMPT_CHOICE_DEFAULT"), - create_color_func("PROMPT_CHOICE_OTHER") - ] - elif colours is None: - colours=[bold] - colours=(colours*len(responses))[:len(responses)] - print(bold(prompt), end=' ') - try: - while True: - if sys.hexversion >= 0x3000000: - response=input("["+"/".join([colours[i](responses[i]) for i in range(len(responses))])+"] ") - else: - response=raw_input("["+"/".join([colours[i](responses[i]) for i in range(len(responses))])+"] ") - if response or not enter_invalid: - for key in responses: - # An empty response will match the - # first value in responses. - if response.upper()==key[:len(response)].upper(): - return key - print("Sorry, response '%s' not understood." % response, end=' ') - except (EOFError, KeyboardInterrupt): - print("Interrupted.") - sys.exit(128 + signal.SIGINT) diff --git a/pym/portage/__init__.py b/pym/portage/__init__.py index d34e06ce7..c24abb0a5 100644 --- a/pym/portage/__init__.py +++ b/pym/portage/__init__.py @@ -494,7 +494,7 @@ _doebuild_manifest_exempt_depend = 0 _testing_eapis = frozenset(["4-python", "4-slot-abi", "5-progress", "5-hdepend"]) _deprecated_eapis = frozenset(["4_pre1", "3_pre2", "3_pre1", "5_pre1", "5_pre2"]) -_supported_eapis = frozenset([str(x) for x in range(portage.const.EAPI)] + list(_testing_eapis) + list(_deprecated_eapis)) +_supported_eapis = frozenset([str(x) for x in range(portage.const.EAPI + 1)] + list(_testing_eapis) + list(_deprecated_eapis)) def _eapi_is_deprecated(eapi): return eapi in _deprecated_eapis @@ -506,19 +506,7 @@ def eapi_is_supported(eapi): eapi = str(eapi) eapi = eapi.strip() - if _eapi_is_deprecated(eapi): - return True - - if eapi in _testing_eapis: - return True - - try: - eapi = int(eapi) - except ValueError: - eapi = -1 - if eapi < 0: - return False - return eapi <= portage.const.EAPI + return eapi in _supported_eapis # This pattern is specified by PMS section 7.3.1. _pms_eapi_re = re.compile(r"^[ \t]*EAPI=(['\"]?)([A-Za-z0-9+_.-]*)\1[ \t]*([ \t]#.*)?$") diff --git a/pym/portage/_emirrordist/FetchTask.py b/pym/portage/_emirrordist/FetchTask.py index 66c41c1a2..307c5bdda 100644 --- a/pym/portage/_emirrordist/FetchTask.py +++ b/pym/portage/_emirrordist/FetchTask.py @@ -1,6 +1,8 @@ -# Copyright 2013 Gentoo Foundation +# Copyright 2013-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import division + import collections import errno import logging @@ -242,7 +244,7 @@ class FetchTask(CompositeTask): remaining_tries = self.config.options.tries - len(self._tried_uris) if remaining_tries > 0: - if remaining_tries <= self.config.options.tries / 2: + if remaining_tries <= self.config.options.tries // 2: while self._primaryuri_stack: uri = self._primaryuri_stack.pop() if uri not in self._tried_uris: diff --git a/pym/portage/_global_updates.py b/pym/portage/_global_updates.py index dde726836..17dc0803b 100644 --- a/pym/portage/_global_updates.py +++ b/pym/portage/_global_updates.py @@ -118,7 +118,9 @@ def _do_global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): else: bindb = None - master_repo = portdb.getRepositoryName(portdb.porttree_root) + master_repo = portdb.repositories.mainRepo() + if master_repo is not None: + master_repo = master_repo.name if master_repo in repo_map: repo_map['DEFAULT'] = repo_map[master_repo] diff --git a/pym/portage/_sets/dbapi.py b/pym/portage/_sets/dbapi.py index 384fb3aa8..299cb8157 100644 --- a/pym/portage/_sets/dbapi.py +++ b/pym/portage/_sets/dbapi.py @@ -1,17 +1,21 @@ -# Copyright 2007-2012 Gentoo Foundation +# Copyright 2007-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import division + +import re import time from portage import os from portage.versions import best, catsplit, vercmp -from portage.dep import Atom +from portage.dep import Atom, use_reduce +from portage.exception import InvalidAtom from portage.localization import _ from portage._sets.base import PackageSet from portage._sets import SetConfigError, get_boolean import portage -__all__ = ["CategorySet", "DowngradeSet", +__all__ = ["CategorySet", "ChangedDepsSet", "DowngradeSet", "EverythingSet", "OwnerSet", "VariableSet"] class EverythingSet(PackageSet): @@ -456,3 +460,78 @@ class RebuiltBinaries(EverythingSet): bindb=trees["bintree"].dbapi) singleBuilder = classmethod(singleBuilder) + +class ChangedDepsSet(PackageSet): + + _operations = ["merge", "unmerge"] + + description = "Package set which contains all installed " + \ + "packages for which the vdb *DEPEND entries are outdated " + \ + "compared to corresponding portdb entries." + + def __init__(self, portdb=None, vardb=None): + super(ChangedDepsSet, self).__init__() + self._portdb = portdb + self._vardb = vardb + + def load(self): + depvars = ('RDEPEND', 'PDEPEND') + + # regexp used to match atoms using subslot operator := + subslot_repl_re = re.compile(r':[^[]*=') + + atoms = [] + for cpv in self._vardb.cpv_all(): + # no ebuild, no update :). + if not self._portdb.cpv_exists(cpv): + continue + + # USE flags used to build the ebuild and EAPI + # (needed for Atom & use_reduce()) + use, eapi = self._vardb.aux_get(cpv, ('USE', 'EAPI')) + usel = use.split() + + # function used to recursively process atoms in nested lists. + def clean_subslots(depatom, usel=None): + if isinstance(depatom, list): + # process the nested list. + return [clean_subslots(x, usel) for x in depatom] + else: + try: + # this can be either an atom or some special operator. + # in the latter case, we get InvalidAtom and pass it as-is. + a = Atom(depatom) + except InvalidAtom: + return depatom + else: + # if we're processing portdb, we need to evaluate USE flag + # dependency conditionals to make them match vdb. this + # requires passing the list of USE flags, so we reuse it + # as conditional for the operation as well. + if usel is not None: + a = a.evaluate_conditionals(usel) + + # replace slot operator := dependencies with plain := + # since we can't properly compare expanded slots + # in vardb to abstract slots in portdb. + return subslot_repl_re.sub(':=', a) + + # get all *DEPEND variables from vdb & portdb and compare them. + # we need to do some cleaning up & expansion to make matching + # meaningful since vdb dependencies are conditional-free. + vdbvars = [clean_subslots(use_reduce(x, uselist=usel, eapi=eapi)) + for x in self._vardb.aux_get(cpv, depvars)] + pdbvars = [clean_subslots(use_reduce(x, uselist=usel, eapi=eapi), usel) + for x in self._portdb.aux_get(cpv, depvars)] + + # if dependencies don't match, trigger the rebuild. + if vdbvars != pdbvars: + atoms.append('=%s' % cpv) + + self._setAtoms(atoms) + + def singleBuilder(cls, options, settings, trees): + return cls(portdb=trees["porttree"].dbapi, + vardb=trees["vartree"].dbapi) + + singleBuilder = classmethod(singleBuilder) diff --git a/pym/portage/cache/sqlite.py b/pym/portage/cache/sqlite.py index 42a239922..310ac9460 100644 --- a/pym/portage/cache/sqlite.py +++ b/pym/portage/cache/sqlite.py @@ -1,7 +1,7 @@ # Copyright 1999-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals +from __future__ import division, unicode_literals import re import sys @@ -174,7 +174,7 @@ class database(fs_template.FsBased): cursor.execute("PRAGMA page_size") page_size=int(cursor.fetchone()[0]) # number of pages, sqlite default is 2000 - cache_size = cache_bytes / page_size + cache_size = cache_bytes // page_size cursor.execute("PRAGMA cache_size = %d" % cache_size) cursor.execute("PRAGMA cache_size") actual_cache_size = int(cursor.fetchone()[0]) diff --git a/pym/portage/const.py b/pym/portage/const.py index 89d7ee29f..5f00fab87 100644 --- a/pym/portage/const.py +++ b/pym/portage/const.py @@ -1,5 +1,5 @@ # portage: Constants -# Copyright 1998-2013 Gentoo Foundation +# Copyright 1998-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import unicode_literals @@ -69,10 +69,10 @@ PORTAGE_BASE_PATH = PORTAGE_BASE # NOTE: Use realpath(__file__) so that python module symlinks in site-packages # are followed back to the real location of the whole portage installation. #PREFIX: below should work, but I'm not sure how it it affects other places -#PORTAGE_BASE_PATH = os.path.join(os.sep, os.sep.join(os.path.realpath( -# __file__.rstrip("co")).split(os.sep)[:-3])) +# NOTE: Please keep PORTAGE_BASE_PATH in one line to help substitutions. +#PORTAGE_BASE_PATH = os.path.join(os.sep, os.sep.join(os.path.realpath(__file__.rstrip("co")).split(os.sep)[:-3])) PORTAGE_BIN_PATH = PORTAGE_BASE_PATH + "/bin" -PORTAGE_PYM_PATH = PORTAGE_BASE_PATH + "/pym" +PORTAGE_PYM_PATH = os.path.realpath(os.path.join(__file__, '../..')) LOCALE_DATA_PATH = PORTAGE_BASE_PATH + "/locale" # FIXME: not used EBUILD_SH_BINARY = PORTAGE_BIN_PATH + "/ebuild.sh" MISC_SH_BINARY = PORTAGE_BIN_PATH + "/misc-functions.sh" @@ -94,6 +94,7 @@ PORTAGE_GROUPNAME = portagegroup PORTAGE_USERNAME = portageuser INVALID_ENV_FILE = "/etc/spork/is/not/valid/profile.env" +MERGING_IDENTIFIER = "-MERGING-" REPO_NAME_FILE = "repo_name" REPO_NAME_LOC = "profiles" + "/" + REPO_NAME_FILE @@ -291,6 +292,9 @@ SUPPORTED_BINPKG_FORMATS = ("tar", "rpm") # Time formats used in various places like metadata.chk. TIMESTAMP_FORMAT = "%a, %d %b %Y %H:%M:%S +0000" # to be used with time.gmtime() +# Top-level names of Python packages installed by Portage. +PORTAGE_PYM_PACKAGES = ("_emerge", "portage", "repoman") + # =========================================================================== # END OF CONSTANTS -- END OF CONSTANTS -- END OF CONSTANTS -- END OF CONSTANT # =========================================================================== diff --git a/pym/portage/dbapi/__init__.py b/pym/portage/dbapi/__init__.py index a20a1e84f..34dfaa760 100644 --- a/pym/portage/dbapi/__init__.py +++ b/pym/portage/dbapi/__init__.py @@ -1,4 +1,4 @@ -# Copyright 1998-2013 Gentoo Foundation +# Copyright 1998-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import unicode_literals @@ -16,6 +16,8 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.versions:catsplit,catpkgsplit,vercmp,_pkg_str', ) +from portage.const import MERGING_IDENTIFIER + from portage import os from portage import auxdbkeys from portage.eapi import _get_eapi_attrs @@ -278,7 +280,7 @@ class dbapi(object): return True def invalidentry(self, mypath): - if '/-MERGING-' in mypath: + if "/" + MERGING_IDENTIFIER in mypath: if os.path.exists(mypath): writemsg(colorize("BAD", _("INCOMPLETE MERGE:"))+" %s\n" % mypath, noiselevel=-1) diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index 040b546fa..deeb77950 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -1,7 +1,7 @@ # Copyright 1998-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from __future__ import unicode_literals +from __future__ import division, unicode_literals __all__ = [ "vardbapi", "vartree", "dblink"] + \ @@ -25,6 +25,7 @@ portage.proxy.lazyimport.lazyimport(globals(), '_merge_unicode_error', '_spawn_phase', 'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs', 'portage.package.ebuild._ipc.QueryCommand:QueryCommand', + 'portage.process:find_binary', 'portage.util:apply_secpass_permissions,ConfigProtect,ensure_dirs,' + \ 'writemsg,writemsg_level,write_atomic,atomic_ofstream,writedict,' + \ 'grabdict,normalize_path,new_protect_filename', @@ -48,7 +49,7 @@ portage.proxy.lazyimport.lazyimport(globals(), ) from portage.const import CACHE_PATH, CONFIG_MEMORY_FILE, \ - PORTAGE_PACKAGE_ATOM, PRIVATE_PATH, VDB_PATH, EPREFIX, EPREFIX_LSTRIP, BASH_BINARY + MERGING_IDENTIFIER, PORTAGE_PACKAGE_ATOM, PRIVATE_PATH, VDB_PATH, EPREFIX, EPREFIX_LSTRIP, BASH_BINARY from portage.dbapi import dbapi from portage.exception import CommandNotFound, \ InvalidData, InvalidLocation, InvalidPackageName, \ @@ -108,7 +109,7 @@ class vardbapi(dbapi): _excluded_dirs = ["CVS", "lost+found"] _excluded_dirs = [re.escape(x) for x in _excluded_dirs] - _excluded_dirs = re.compile(r'^(\..*|-MERGING-.*|' + \ + _excluded_dirs = re.compile(r'^(\..*|' + MERGING_IDENTIFIER + '.*|' + \ "|".join(_excluded_dirs) + r')$') _aux_cache_version = "1" @@ -1058,7 +1059,7 @@ class vardbapi(dbapi): from md5 import new as _new_hash _hash_bits = 16 - _hex_chars = int(_hash_bits / 4) + _hex_chars = _hash_bits // 4 def __init__(self, vardb): self._vardb = vardb @@ -1519,7 +1520,7 @@ class dblink(object): self.dbroot = normalize_path(os.path.join(self._eroot, VDB_PATH)) self.dbcatdir = self.dbroot+"/"+cat self.dbpkgdir = self.dbcatdir+"/"+pkg - self.dbtmpdir = self.dbcatdir+"/-MERGING-"+pkg + self.dbtmpdir = self.dbcatdir+"/"+MERGING_IDENTIFIER+pkg self.dbdir = self.dbpkgdir self.settings = mysettings self._verbose = self.settings.get("PORTAGE_VERBOSE") == "1" @@ -3746,7 +3747,7 @@ class dblink(object): break relative_path = parent[srcroot_len:] - dirlist.append(os.path.join("/", relative_path)) + dirlist.append(os.path.join(destroot, relative_path)) for fname in files: try: @@ -3872,8 +3873,7 @@ class dblink(object): msg = textwrap.wrap(msg, 70) msg.append("") for f in rofilesystems: - msg.append("\t%s" % os.path.join(destroot, - f.lstrip(os.path.sep))) + msg.append("\t%s" % f) msg.append("") self._elog("eerror", "preinst", msg) @@ -5035,6 +5035,15 @@ class dblink(object): quickpkg_binary = os.path.join(self.settings["PORTAGE_BIN_PATH"], "quickpkg") + if not os.access(quickpkg_binary, os.X_OK): + # If not running from the source tree, use PATH. + quickpkg_binary = find_binary("quickpkg") + if quickpkg_binary is None: + self._display_merge( + _("%s: command not found") % "quickpkg", + level=logging.ERROR, noiselevel=-1) + return 127 + # Let quickpkg inherit the global vartree config's env. env = dict(self.vartree.settings.items()) env["__PORTAGE_INHERIT_VARDB_LOCK"] = "1" diff --git a/pym/portage/dep/_slot_operator.py b/pym/portage/dep/_slot_operator.py index 7b6444403..8b67fc569 100644 --- a/pym/portage/dep/_slot_operator.py +++ b/pym/portage/dep/_slot_operator.py @@ -1,4 +1,4 @@ -# Copyright 2012-2013 Gentoo Foundation +# Copyright 2012-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import unicode_literals @@ -67,6 +67,11 @@ def evaluate_slot_operator_equal_deps(settings, use, trees): return result def _eval_deps(dep_struct, vardbs): + # TODO: we'd use a better || () handling, i.e. || ( A:= B:= ) with both A + # and B installed should record subslot on A only since the package is + # supposed to link against that anyway, and we have no guarantee that B + # has matching ABI. + for i, x in enumerate(dep_struct): if isinstance(x, list): _eval_deps(x, vardbs) @@ -87,11 +92,15 @@ def _eval_deps(dep_struct, vardbs): dep_struct[i] = x break else: - # this dep could not be resolved, so remove the operator - # (user may be using package.provided and managing rebuilds - # manually) - if x.slot: - x = x.with_slot(x.slot) - else: - x = x.without_slot - dep_struct[i] = x + # this dep could not be resolved, possibilities include: + # 1. unsatisfied branch of || () dep, + # 2. package.provided, + # 3. --nodeps. + # + # just leave it as-is for now. this does not cause any special + # behavior while keeping the information in vdb -- necessary + # e.g. for @changed-deps to work properly. + # + # TODO: make it actually cause subslot rebuilds when switching + # || () branches. + pass diff --git a/pym/portage/dep/dep_check.py b/pym/portage/dep/dep_check.py index b79c5bc31..4386b5ed1 100644 --- a/pym/portage/dep/dep_check.py +++ b/pym/portage/dep/dep_check.py @@ -1,4 +1,4 @@ -# Copyright 2010-2013 Gentoo Foundation +# Copyright 2010-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import unicode_literals @@ -287,6 +287,7 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): unsat_use_non_installed = [] other_installed = [] other_installed_some = [] + other_installed_any_slot = [] other = [] # unsat_use_* must come after preferred_non_installed @@ -301,6 +302,7 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): unsat_use_non_installed, other_installed, other_installed_some, + other_installed_any_slot, other, ) @@ -414,16 +416,16 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): unsat_use_non_installed.append(this_choice) else: all_in_graph = True - for slot_atom in slot_map: + for atom in atoms: # New-style virtuals have zero cost to install. - if slot_atom.startswith("virtual/"): + if atom.blocker or atom.cp.startswith("virtual/"): continue # We check if the matched package has actually been # added to the digraph, in order to distinguish between # those packages and installed packages that may need # to be uninstalled in order to resolve blockers. - graph_matches = graph_db.match_pkgs(slot_atom) - if not graph_matches or graph_matches[-1] not in graph: + if not any(pkg in graph for pkg in + graph_db.match_pkgs(atom)): all_in_graph = False break circular_atom = None @@ -504,6 +506,14 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): other_installed.append(this_choice) elif some_installed: other_installed_some.append(this_choice) + + # Use Atom(atom.cp) for a somewhat "fuzzy" match, since + # the whole atom may be too specific. For example, see + # bug #522652, where using the whole atom leads to an + # unsatisfiable choice. + elif any(vardb.match(Atom(atom.cp)) for atom in atoms + if not atom.blocker): + other_installed_any_slot.append(this_choice) else: other.append(this_choice) diff --git a/pym/portage/dispatch_conf.py b/pym/portage/dispatch_conf.py index 479c19105..bf6c2ca94 100644 --- a/pym/portage/dispatch_conf.py +++ b/pym/portage/dispatch_conf.py @@ -208,4 +208,5 @@ def rcs_archive_post_process(archive): def file_archive_post_process(archive): """Rename the archive file with the .dist.new suffix to a .dist suffix""" - os.rename(archive + '.dist.new', archive + '.dist') + if os.path.exists(archive + '.dist.new'): + os.rename(archive + '.dist.new', archive + '.dist') diff --git a/pym/portage/emaint/main.py b/pym/portage/emaint/main.py index 6a17027b5..646883d73 100644 --- a/pym/portage/emaint/main.py +++ b/pym/portage/emaint/main.py @@ -98,10 +98,11 @@ def module_opts(module_controller, module): class TaskHandler(object): """Handles the running of the tasks it is given""" - def __init__(self, show_progress_bar=True, verbose=True, callback=None): + def __init__(self, show_progress_bar=True, verbose=True, callback=None, module_output=None): self.show_progress_bar = show_progress_bar self.verbose = verbose self.callback = callback + self.module_output = module_output self.isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty() self.progress_bar = ProgressBar(self.isatty, title="Emaint", max_desc_length=27) @@ -124,6 +125,7 @@ class TaskHandler(object): onProgress = None kwargs = { 'onProgress': onProgress, + 'module_output': self.module_output, # pass in a copy of the options so a module can not pollute or change # them for other tasks if there is more to do. 'options': options.copy() @@ -219,5 +221,5 @@ def emaint_main(myargv): # need to pass the parser options dict to the modules # so they are available if needed. task_opts = options.__dict__ - taskmaster = TaskHandler(callback=print_results) + taskmaster = TaskHandler(callback=print_results, module_output=sys.stdout) taskmaster.run_tasks(tasks, func, status, options=task_opts) diff --git a/pym/portage/emaint/module.py b/pym/portage/emaint/module.py index bf7d25fc5..07a0cb763 100644 --- a/pym/portage/emaint/module.py +++ b/pym/portage/emaint/module.py @@ -66,7 +66,7 @@ class Module(object): kid['is_imported'] = True except ImportError: raise - mod_class = getattr(module, kid['class']) + mod_class = getattr(module, kid['class']) return mod_class diff --git a/pym/portage/emaint/modules/binhost/__init__.py b/pym/portage/emaint/modules/binhost/__init__.py index c60e8bcb4..f2220e9d0 100644 --- a/pym/portage/emaint/modules/binhost/__init__.py +++ b/pym/portage/emaint/modules/binhost/__init__.py @@ -1,18 +1,18 @@ # Copyright 2005-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -"""Scan and generate metadata indexes for binary packages. -""" +doc = """Scan and generate metadata indexes for binary packages.""" +__doc__ = doc module_spec = { 'name': 'binhost', - 'description': __doc__, + 'description': doc, 'provides':{ 'module1': { 'name': "binhost", 'class': "BinhostHandler", - 'description': __doc__, + 'description': doc, 'functions': ['check', 'fix'], 'func_desc': {} } diff --git a/pym/portage/emaint/modules/config/__init__.py b/pym/portage/emaint/modules/config/__init__.py index f0585b39a..0277d3998 100644 --- a/pym/portage/emaint/modules/config/__init__.py +++ b/pym/portage/emaint/modules/config/__init__.py @@ -1,18 +1,18 @@ # Copyright 2005-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -"""Check and clean the config tracker list for uninstalled packages. -""" +doc = """Check and clean the config tracker list for uninstalled packages.""" +__doc__ = doc module_spec = { 'name': 'config', - 'description': __doc__, + 'description': doc, 'provides':{ 'module1': { 'name': "cleanconfmem", 'class': "CleanConfig", - 'description': __doc__, + 'description': doc, 'functions': ['check', 'fix'], 'func_desc': {} } diff --git a/pym/portage/emaint/modules/logs/__init__.py b/pym/portage/emaint/modules/logs/__init__.py index 0407efe2b..a7891fd5b 100644 --- a/pym/portage/emaint/modules/logs/__init__.py +++ b/pym/portage/emaint/modules/logs/__init__.py @@ -1,18 +1,18 @@ # Copyright 2005-2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -"""Check and clean old logs in the PORT_LOGDIR. -""" +doc = """Check and clean old logs in the PORT_LOGDIR.""" +__doc__ = doc module_spec = { 'name': 'logs', - 'description': __doc__, + 'description': doc, 'provides':{ 'module1': { 'name': "logs", 'class': "CleanLogs", - 'description': __doc__, + 'description': doc, 'functions': ['check','clean'], 'func_desc': { 'clean': { diff --git a/pym/portage/emaint/modules/merges/__init__.py b/pym/portage/emaint/modules/merges/__init__.py new file mode 100644 index 000000000..bcb2ac8ee --- /dev/null +++ b/pym/portage/emaint/modules/merges/__init__.py @@ -0,0 +1,31 @@ +# Copyright 2005-2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +doc = """Scan for failed merges and fix them.""" +__doc__ = doc + + +module_spec = { + 'name': 'merges', + 'description': doc, + 'provides': { + 'merges': { + 'name': "merges", + 'class': "MergesHandler", + 'description': doc, + 'functions': ['check', 'fix', 'purge'], + 'func_desc': { + 'purge': { + 'short': '-P', 'long': '--purge-tracker', + 'help': 'Removes the list of previously failed merges.' + + ' WARNING: Only use this option if you plan on' + + ' manually fixing them or do not want them' + ' re-installed.', + 'status': "Removing %s", + 'action': 'store_true', + 'func': 'purge' + } + } + } + } +} diff --git a/pym/portage/emaint/modules/merges/merges.py b/pym/portage/emaint/modules/merges/merges.py new file mode 100644 index 000000000..1a67cb5df --- /dev/null +++ b/pym/portage/emaint/modules/merges/merges.py @@ -0,0 +1,290 @@ +# Copyright 2005-2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from _emerge.actions import load_emerge_config + +import portage +from portage import os, _unicode_encode +from portage.const import MERGING_IDENTIFIER, PORTAGE_BIN_PATH, PRIVATE_PATH, \ + VDB_PATH +from portage.dep import isvalidatom + +import shutil +import subprocess +import sys +import time + +class TrackingFile(object): + """File for keeping track of failed merges.""" + + + def __init__(self, tracking_path): + """ + Create a TrackingFile object. + + @param tracking_path: file path used to keep track of failed merges + @type tracking_path: String + """ + self._tracking_path = _unicode_encode(tracking_path) + + + def save(self, failed_pkgs): + """ + Save the specified packages that failed to merge. + + @param failed_pkgs: dictionary of failed packages + @type failed_pkgs: dict + """ + tracking_path = self._tracking_path + lines = ['%s %s' % (pkg, mtime) for pkg, mtime in failed_pkgs.items()] + portage.util.write_atomic(tracking_path, '\n'.join(lines)) + + + def load(self): + """ + Load previously failed merges. + + @rtype: dict + @return: dictionary of packages that failed to merge + """ + tracking_path = self._tracking_path + if not self.exists(): + return {} + failed_pkgs = {} + with open(tracking_path, 'r') as tracking_file: + for failed_merge in tracking_file: + pkg, mtime = failed_merge.strip().split() + failed_pkgs[pkg] = mtime + return failed_pkgs + + + def exists(self): + """ + Check if tracking file exists. + + @rtype: bool + @return: true if tracking file exists, false otherwise + """ + return os.path.exists(self._tracking_path) + + + def purge(self): + """Delete previously saved tracking file if one exists.""" + if self.exists(): + os.remove(self._tracking_path) + + + def __iter__(self): + """ + Provide an interator over failed merges. + + @return: iterator of packages that failed to merge + """ + return self.load().items().__iter__() + + +class MergesHandler(object): + """Handle failed package merges.""" + + short_desc = "Remove failed merges" + + @staticmethod + def name(): + return "merges" + + + def __init__(self): + """Create MergesHandler object.""" + eroot = portage.settings['EROOT'] + tracking_path = os.path.join(eroot, PRIVATE_PATH, 'failed-merges'); + self._tracking_file = TrackingFile(tracking_path) + self._vardb_path = os.path.join(eroot, VDB_PATH) + + + def can_progressbar(self, func): + return func == 'check' + + + def _scan(self, onProgress=None): + """ + Scan the file system for failed merges and return any found. + + @param onProgress: function to call for updating progress + @type onProgress: Function + @rtype: dict + @return: dictionary of packages that failed to merges + """ + failed_pkgs = {} + for cat in os.listdir(self._vardb_path): + pkgs_path = os.path.join(self._vardb_path, cat) + if not os.path.isdir(pkgs_path): + continue + pkgs = os.listdir(pkgs_path) + maxval = len(pkgs) + for i, pkg in enumerate(pkgs): + if onProgress: + onProgress(maxval, i+1) + if MERGING_IDENTIFIER in pkg: + mtime = int(os.stat(os.path.join(pkgs_path, pkg)).st_mtime) + pkg = os.path.join(cat, pkg) + failed_pkgs[pkg] = mtime + return failed_pkgs + + + def _failed_pkgs(self, onProgress=None): + """ + Return failed packages from both the file system and tracking file. + + @rtype: dict + @return: dictionary of packages that failed to merges + """ + failed_pkgs = self._scan(onProgress) + for pkg, mtime in self._tracking_file: + if pkg not in failed_pkgs: + failed_pkgs[pkg] = mtime + return failed_pkgs + + + def _remove_failed_dirs(self, failed_pkgs): + """ + Remove the directories of packages that failed to merge. + + @param failed_pkgs: failed packages whose directories to remove + @type failed_pkg: dict + """ + for failed_pkg in failed_pkgs: + pkg_path = os.path.join(self._vardb_path, failed_pkg) + # delete failed merge directory if it exists (it might not exist + # if loaded from tracking file) + if os.path.exists(pkg_path): + shutil.rmtree(pkg_path) + # TODO: try removing package CONTENTS to prevent orphaned + # files + + + def _get_pkg_atoms(self, failed_pkgs, pkg_atoms, pkg_invalid_entries): + """ + Get the package atoms for the specified failed packages. + + @param failed_pkgs: failed packages to iterate + @type failed_pkgs: dict + @param pkg_atoms: add package atoms to this set + @type pkg_atoms: set + @param pkg_invalid_entries: add any packages that are invalid to this set + @type pkg_invalid_entries: set + """ + + emerge_config = load_emerge_config() + portdb = emerge_config.target_config.trees['porttree'].dbapi + for failed_pkg in failed_pkgs: + # validate pkg name + pkg_name = '%s' % failed_pkg.replace(MERGING_IDENTIFIER, '') + pkg_atom = '=%s' % pkg_name + + if not isvalidatom(pkg_atom): + pkg_invalid_entries.add("'%s' is an invalid package atom." + % pkg_atom) + if not portdb.cpv_exists(pkg_name): + pkg_invalid_entries.add( + "'%s' does not exist in the portage tree." % pkg_name) + pkg_atoms.add(pkg_atom) + + + def _emerge_pkg_atoms(self, module_output, pkg_atoms): + """ + Emerge the specified packages atoms. + + @param module_output: output will be written to + @type module_output: Class + @param pkg_atoms: packages atoms to emerge + @type pkg_atoms: set + @rtype: list + @return: List of results + """ + # TODO: rewrite code to use portage's APIs instead of a subprocess + env = { + "FEATURES" : "-collision-protect -protect-owned", + "PATH" : os.environ["PATH"] + } + emerge_cmd = ( + portage._python_interpreter, + '-b', + os.path.join(PORTAGE_BIN_PATH, 'emerge'), + '--quiet', + '--oneshot', + '--complete-graph=y' + ) + results = [] + msg = 'Re-Emerging packages that failed to merge...\n' + if module_output: + module_output.write(msg) + else: + module_output = subprocess.PIPE + results.append(msg) + proc = subprocess.Popen(emerge_cmd + tuple(pkg_atoms), env=env, + stdout=module_output, stderr=sys.stderr) + output = proc.communicate()[0] + if output: + results.append(output) + if proc.returncode != os.EX_OK: + emerge_status = "Failed to emerge '%s'" % (' '.join(pkg_atoms)) + else: + emerge_status = "Successfully emerged '%s'" % (' '.join(pkg_atoms)) + results.append(emerge_status) + return results + + + def check(self, **kwargs): + """Check for failed merges.""" + onProgress = kwargs.get('onProgress', None) + failed_pkgs = self._failed_pkgs(onProgress) + errors = [] + for pkg, mtime in failed_pkgs.items(): + mtime_str = time.ctime(int(mtime)) + errors.append("'%s' failed to merge on '%s'" % (pkg, mtime_str)) + return errors + + + def fix(self, **kwargs): + """Attempt to fix any failed merges.""" + module_output = kwargs.get('module_output', None) + failed_pkgs = self._failed_pkgs() + if not failed_pkgs: + return ['No failed merges found.'] + + pkg_invalid_entries = set() + pkg_atoms = set() + self._get_pkg_atoms(failed_pkgs, pkg_atoms, pkg_invalid_entries) + if pkg_invalid_entries: + return pkg_invalid_entries + + try: + self._tracking_file.save(failed_pkgs) + except IOError as ex: + errors = ['Unable to save failed merges to tracking file: %s\n' + % str(ex)] + errors.append(', '.join(sorted(failed_pkgs))) + return errors + self._remove_failed_dirs(failed_pkgs) + results = self._emerge_pkg_atoms(module_output, pkg_atoms) + # list any new failed merges + for pkg in sorted(self._scan()): + results.append("'%s' still found as a failed merge." % pkg) + # reload config and remove successful packages from tracking file + emerge_config = load_emerge_config() + vardb = emerge_config.target_config.trees['vartree'].dbapi + still_failed_pkgs = {} + for pkg, mtime in failed_pkgs.items(): + pkg_name = '%s' % pkg.replace(MERGING_IDENTIFIER, '') + if not vardb.cpv_exists(pkg_name): + still_failed_pkgs[pkg] = mtime + self._tracking_file.save(still_failed_pkgs) + return results + + + def purge(self, **kwargs): + """Attempt to remove previously saved tracking file.""" + if not self._tracking_file.exists(): + return ['Tracking file not found.'] + self._tracking_file.purge() + return ['Removed tracking file.'] diff --git a/pym/portage/emaint/modules/move/__init__.py b/pym/portage/emaint/modules/move/__init__.py index d31d7b346..51624306c 100644 --- a/pym/portage/emaint/modules/move/__init__.py +++ b/pym/portage/emaint/modules/move/__init__.py @@ -1,18 +1,18 @@ # Copyright 2005-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -"""Perform package move updates for installed and binary packages. -""" +doc = """Perform package move updates for installed and binary packages.""" +__doc__ = doc module_spec = { 'name': 'move', - 'description': __doc__, + 'description': doc, 'provides':{ 'module1': { 'name': "moveinst", 'class': "MoveInstalled", - 'description': __doc__, + 'description': doc, 'options': ['check', 'fix'], 'functions': ['check', 'fix'], 'func_desc': { diff --git a/pym/portage/emaint/modules/move/move.py b/pym/portage/emaint/modules/move/move.py index ef674d47a..41ca167b7 100644 --- a/pym/portage/emaint/modules/move/move.py +++ b/pym/portage/emaint/modules/move/move.py @@ -13,8 +13,9 @@ class MoveHandler(object): self._tree = tree self._portdb = porttree.dbapi self._update_keys = Package._dep_keys + ("PROVIDE",) - self._master_repo = \ - self._portdb.getRepositoryName(self._portdb.porttree_root) + self._master_repo = self._portdb.repositories.mainRepo() + if self._master_repo is not None: + self._master_repo = self._master_repo.name def _grab_global_updates(self): from portage.update import grab_updates, parse_updates diff --git a/pym/portage/emaint/modules/resume/__init__.py b/pym/portage/emaint/modules/resume/__init__.py index 965e8f945..ebe4a37df 100644 --- a/pym/portage/emaint/modules/resume/__init__.py +++ b/pym/portage/emaint/modules/resume/__init__.py @@ -1,13 +1,13 @@ # Copyright 2005-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -"""Check and fix problems in the resume and/or resume_backup files. -""" +doc = """Check and fix problems in the resume and/or resume_backup files.""" +__doc__ = doc module_spec = { 'name': 'resume', - 'description': __doc__, + 'description': doc, 'provides':{ 'module1': { 'name': "cleanresume", diff --git a/pym/portage/emaint/modules/world/__init__.py b/pym/portage/emaint/modules/world/__init__.py index 3f62270ee..0af73d4e8 100644 --- a/pym/portage/emaint/modules/world/__init__.py +++ b/pym/portage/emaint/modules/world/__init__.py @@ -1,18 +1,18 @@ # Copyright 2005-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -"""Check and fix problems in the world file. -""" +doc = """Check and fix problems in the world file.""" +__doc__ = doc module_spec = { 'name': 'world', - 'description': __doc__, + 'description': doc, 'provides':{ 'module1':{ 'name': "world", 'class': "WorldHandler", - 'description': __doc__, + 'description': doc, 'functions': ['check', 'fix'], 'func_desc': {} } diff --git a/pym/portage/exception.py b/pym/portage/exception.py index 6fa5447a7..ef62e7ae1 100644 --- a/pym/portage/exception.py +++ b/pym/portage/exception.py @@ -84,6 +84,10 @@ class FileNotFound(InvalidLocation): class DirectoryNotFound(InvalidLocation): """A directory was not found when it was expected to exist""" +class IsADirectory(PortageException): + """A directory was found when it was expected to be a file""" + from errno import EISDIR as errno + class OperationNotPermitted(PortageException): """An operation was not permitted operating system""" from errno import EPERM as errno diff --git a/pym/portage/localization.py b/pym/portage/localization.py index e4d87b65b..2db4b7ada 100644 --- a/pym/portage/localization.py +++ b/pym/portage/localization.py @@ -2,10 +2,12 @@ # Copyright 2004-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import division + import locale import math -from portage import _unicode_decode +from portage import _encodings, _unicode_decode # We define this to make the transition easier for us. def _(mystr): @@ -36,4 +38,5 @@ def localized_size(num_bytes): # always round up, so that small files don't end up as '0 KiB' num_kib = math.ceil(num_bytes / 1024) - return locale.format('%d', num_kib, grouping=True) + ' KiB' + formatted_num = locale.format('%d', num_kib, grouping=True) + return (_unicode_decode(formatted_num, encoding=_encodings['stdio']) + ' KiB') diff --git a/pym/portage/mail.py b/pym/portage/mail.py index 723da04b8..11923eea6 100644 --- a/pym/portage/mail.py +++ b/pym/portage/mail.py @@ -12,7 +12,6 @@ import socket import sys -import time from portage import os from portage import _encodings @@ -49,6 +48,7 @@ def create_message(sender, recipient, subject, body, attachments=None): from email.header import Header from email.mime.base import MIMEBase as BaseMessage from email.mime.multipart import MIMEMultipart as MultipartMessage + from email.utils import formatdate if sys.hexversion < 0x3000000: sender = _unicode_encode(sender, @@ -91,8 +91,8 @@ def create_message(sender, recipient, subject, body, attachments=None): # input_bytes = s.encode(input_charset, errors) #UnicodeEncodeError: 'ascii' codec can't encode characters in position 0-9: ordinal not in range(128) mymessage["Subject"] = Header(_force_ascii_if_necessary(subject)) - mymessage["Date"] = time.strftime("%a, %d %b %Y %H:%M:%S %z") - + mymessage["Date"] = formatdate(localtime=True) + return mymessage def send_mail(mysettings, message): @@ -104,7 +104,7 @@ def send_mail(mysettings, message): mymailuser = "" mymailpasswd = "" myrecipient = "root@localhost" - + # Syntax for PORTAGE_ELOG_MAILURI (if defined): # address [[user:passwd@]mailserver[:port]] # where address: recipient address @@ -129,7 +129,7 @@ def send_mail(mysettings, message): mymailhost = myconndata else: myrecipient = mysettings.get("PORTAGE_ELOG_MAILURI", "") - + myfrom = message.get("From") if sys.hexversion < 0x3000000: @@ -174,4 +174,4 @@ def send_mail(mysettings, message): except socket.error as e: raise portage.exception.PortageException(_("!!! A network error occurred while trying to send logmail:\n%s\nSure you configured PORTAGE_ELOG_MAILURI correctly?") % str(e)) return - + diff --git a/pym/portage/news.py b/pym/portage/news.py index 408fb5c5f..0d72b000d 100644 --- a/pym/portage/news.py +++ b/pym/portage/news.py @@ -61,10 +61,12 @@ class NewsManager(object): self._dir_mode = 0o0074 self._mode_mask = 0o0000 - portdir = portdb.porttree_root - profiles_base = os.path.join(portdir, 'profiles') + os.path.sep + portdir = portdb.repositories.mainRepoLocation() + profiles_base = None + if portdir is not None: + profiles_base = os.path.join(portdir, 'profiles') + os.path.sep profile_path = None - if portdb.settings.profile_path: + if profiles_base is not None and portdb.settings.profile_path: profile_path = normalize_path( os.path.realpath(portdb.settings.profile_path)) if profile_path.startswith(profiles_base): diff --git a/pym/portage/output.py b/pym/portage/output.py index abe6b63cc..785168716 100644 --- a/pym/portage/output.py +++ b/pym/portage/output.py @@ -1,6 +1,8 @@ # Copyright 1998-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import division + __docformat__ = "epytext" import errno @@ -778,14 +780,14 @@ class TermProgressBar(ProgressBar): "<=>" + ((max_bar_width - bar_width) * " ") + "]") return image else: - percentage = int(100 * float(curval) / maxval) + percentage = 100 * curval // maxval max_bar_width = bar_space - 1 _percent = ("%d%% " % percentage).rjust(percentage_str_width) image = "%s%s" % (self._desc, _percent) if cols < min_columns: return image - offset = float(curval) / maxval + offset = curval / maxval bar_width = int(offset * max_bar_width) image = image + "[" + (bar_width * "=") + \ ">" + ((max_bar_width - bar_width) * " ") + "]" diff --git a/pym/portage/package/ebuild/config.py b/pym/portage/package/ebuild/config.py index fb4956d5d..6e578a9e1 100644 --- a/pym/portage/package/ebuild/config.py +++ b/pym/portage/package/ebuild/config.py @@ -37,7 +37,8 @@ from portage.dep import Atom, isvalidatom, match_from_list, use_reduce, _repo_se from portage.eapi import eapi_exports_AA, eapi_exports_merge_type, \ eapi_supports_prefix, eapi_exports_replace_vars, _get_eapi_attrs from portage.env.loaders import KeyValuePairFileLoader -from portage.exception import InvalidDependString, PortageException +from portage.exception import InvalidDependString, IsADirectory, \ + PortageException from portage.localization import _ from portage.output import colorize from portage.process import fakeroot_capable, sandbox_capable, macossandbox_capable @@ -556,8 +557,14 @@ class config(object): self.profile_path = locations_manager.profile_path self.user_profile_dir = locations_manager.user_profile_dir - packages_list = [grabfile_package(os.path.join(x, "packages"), - verify_eapi=True) for x in self.profiles] + try: + packages_list = [grabfile_package(os.path.join(x, "packages"), + verify_eapi=True) for x in self.profiles] + except IOError as e: + if e.errno == IsADirectory.errno: + raise IsADirectory(os.path.join(self.profile_path, + "packages")) + self.packages = tuple(stack_lists(packages_list, incremental=1)) # revmaskdict @@ -1359,6 +1366,7 @@ class config(object): previous_iuse = pkg_configdict.get("IUSE") previous_iuse_effective = pkg_configdict.get("IUSE_EFFECTIVE") previous_features = pkg_configdict.get("FEATURES") + previous_penv = self._penv aux_keys = self._setcpv_aux_keys @@ -1526,6 +1534,9 @@ class config(object): else: pkg_configdict['USE'] = self.puse + elif previous_penv: + has_changed = True + if has_changed: self.reset(keeping_pkg=1) diff --git a/pym/portage/package/ebuild/doebuild.py b/pym/portage/package/ebuild/doebuild.py index 3c2167a03..8e55fe23e 100644 --- a/pym/portage/package/ebuild/doebuild.py +++ b/pym/portage/package/ebuild/doebuild.py @@ -46,8 +46,7 @@ from portage import auxdbkeys, bsd_chflags, \ unmerge, _encodings, _os_merge, \ _shell_quote, _unicode_decode, _unicode_encode from portage.const import EBUILD_SH_ENV_FILE, EBUILD_SH_ENV_DIR, \ - EBUILD_SH_BINARY, INVALID_ENV_FILE, MISC_SH_BINARY, \ - EPREFIX, MACOSSANDBOX_PROFILE + EBUILD_SH_BINARY, INVALID_ENV_FILE, MISC_SH_BINARY, PORTAGE_PYM_PACKAGES, EPREFIX, MACOSSANDBOX_PROFILE from portage.data import portage_gid, portage_uid, secpass, \ uid, userpriv_groups from portage.dbapi.porttree import _parse_uri_map @@ -2407,7 +2406,11 @@ def _prepare_self_update(settings): orig_pym_path = portage._pym_path portage._pym_path = os.path.join(base_path_tmp, "pym") - shutil.copytree(orig_pym_path, portage._pym_path, symlinks=True) + os.mkdir(portage._pym_path) + for pmod in PORTAGE_PYM_PACKAGES: + shutil.copytree(os.path.join(orig_pym_path, pmod), + os.path.join(portage._pym_path, pmod), + symlinks=True) for dir_path in (base_path_tmp, portage._bin_path, portage._pym_path): os.chmod(dir_path, 0o755) diff --git a/pym/portage/tests/__init__.py b/pym/portage/tests/__init__.py index 84e732a1c..708dd5934 100644 --- a/pym/portage/tests/__init__.py +++ b/pym/portage/tests/__init__.py @@ -25,8 +25,24 @@ import portage from portage import os from portage import _encodings from portage import _unicode_decode +from portage.const import (EPREFIX, GLOBAL_CONFIG_PATH, PORTAGE_BASE_PATH, + PORTAGE_BIN_PATH) from portage.util._argparse import ArgumentParser + +if portage._not_installed: + cnf_path = os.path.join(PORTAGE_BASE_PATH, 'cnf') + cnf_etc_path = cnf_path + cnf_bindir = PORTAGE_BIN_PATH + cnf_sbindir = cnf_bindir +else: + cnf_path = os.path.join(EPREFIX or '/', GLOBAL_CONFIG_PATH) + cnf_etc_path = os.path.join(EPREFIX or '/', 'etc') + cnf_eprefix = EPREFIX + cnf_bindir = os.path.join(EPREFIX or '/', 'usr', 'bin') + cnf_sbindir = os.path.join(EPREFIX or '/', 'usr', 'sbin') + + def main(): suite = unittest.TestSuite() basedir = os.path.dirname(os.path.realpath(__file__)) @@ -83,11 +99,11 @@ def getTestFromCommandLine(args, base_path): return result def getTestDirs(base_path): - TEST_FILE = b'__test__' + TEST_FILE = b'__test__.py' testDirs = [] # the os.walk help mentions relative paths as being quirky - # I was tired of adding dirs to the list, so now we add __test__ + # I was tired of adding dirs to the list, so now we add __test__.py # to each dir we want tested. for root, dirs, files in os.walk(base_path): try: @@ -178,6 +194,10 @@ class TestCase(unittest.TestCase): unittest.TestCase.__init__(self, *pargs, **kwargs) self.todo = False self.portage_skip = None + self.cnf_path = cnf_path + self.cnf_etc_path = cnf_etc_path + self.bindir = cnf_bindir + self.sbindir = cnf_sbindir def defaultTestResult(self): return TextTestResult() diff --git a/doc/fragment/date b/pym/portage/tests/bin/__test__.py index e69de29bb..e69de29bb 100644 --- a/doc/fragment/date +++ b/pym/portage/tests/bin/__test__.py diff --git a/pym/portage/tests/bin/__test__ b/pym/portage/tests/dbapi/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/bin/__test__ +++ b/pym/portage/tests/dbapi/__test__.py diff --git a/pym/portage/tests/dbapi/test_portdb_cache.py b/pym/portage/tests/dbapi/test_portdb_cache.py index 94af96eaf..f08d0f80b 100644 --- a/pym/portage/tests/dbapi/test_portdb_cache.py +++ b/pym/portage/tests/dbapi/test_portdb_cache.py @@ -8,8 +8,7 @@ import textwrap import portage from portage import os from portage import _unicode_decode -from portage.const import (BASH_BINARY, PORTAGE_BIN_PATH, - PORTAGE_PYM_PATH, USER_CONFIG_PATH) +from portage.const import (BASH_BINARY, PORTAGE_PYM_PATH, USER_CONFIG_PATH) from portage.tests import TestCase from portage.tests.resolver.ResolverPlayground import ResolverPlayground from portage.util import ensure_dirs @@ -38,7 +37,7 @@ class PortdbCacheTestCase(TestCase): portage_python = portage._python_interpreter egencache_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "egencache"), + os.path.join(self.bindir, "egencache"), "--repo", "test_repo", "--repositories-configuration", settings.repositories.config_string()) python_cmd = (portage_python, "-b", "-Wd", "-c") @@ -48,7 +47,7 @@ class PortdbCacheTestCase(TestCase): (lambda: not os.path.exists(md5_cache_dir),), python_cmd + (textwrap.dedent(""" import os, sys, portage - if portage.portdb.porttree_root in portage.portdb._pregen_auxdb: + if portage.portdb.repositories.mainRepoLocation() in portage.portdb._pregen_auxdb: sys.exit(1) """),), @@ -57,13 +56,13 @@ class PortdbCacheTestCase(TestCase): (lambda: os.path.exists(md5_cache_dir),), python_cmd + (textwrap.dedent(""" import os, sys, portage - if portage.portdb.porttree_root not in portage.portdb._pregen_auxdb: + if portage.portdb.repositories.mainRepoLocation() not in portage.portdb._pregen_auxdb: sys.exit(1) """),), python_cmd + (textwrap.dedent(""" import os, sys, portage from portage.cache.flat_hash import md5_database - if not isinstance(portage.portdb._pregen_auxdb[portage.portdb.porttree_root], md5_database): + if not isinstance(portage.portdb._pregen_auxdb[portage.portdb.repositories.mainRepoLocation()], md5_database): sys.exit(1) """),), @@ -74,13 +73,13 @@ class PortdbCacheTestCase(TestCase): (lambda: os.path.exists(md5_cache_dir),), python_cmd + (textwrap.dedent(""" import os, sys, portage - if portage.portdb.porttree_root not in portage.portdb._pregen_auxdb: + if portage.portdb.repositories.mainRepoLocation() not in portage.portdb._pregen_auxdb: sys.exit(1) """),), python_cmd + (textwrap.dedent(""" import os, sys, portage from portage.cache.flat_hash import md5_database - if not isinstance(portage.portdb._pregen_auxdb[portage.portdb.porttree_root], md5_database): + if not isinstance(portage.portdb._pregen_auxdb[portage.portdb.repositories.mainRepoLocation()], md5_database): sys.exit(1) """),), @@ -91,13 +90,13 @@ class PortdbCacheTestCase(TestCase): ("cache-formats = pms md5-dict", layout_conf_path,)))), (portage_python, "-b", "-Wd", "-Wi::DeprecationWarning", "-c") + (textwrap.dedent(""" import os, sys, portage - if portage.portdb.porttree_root not in portage.portdb._pregen_auxdb: + if portage.portdb.repositories.mainRepoLocation() not in portage.portdb._pregen_auxdb: sys.exit(1) """),), (portage_python, "-b", "-Wd", "-Wi::DeprecationWarning", "-c") + (textwrap.dedent(""" import os, sys, portage from portage.cache.metadata import database as pms_database - if not isinstance(portage.portdb._pregen_auxdb[portage.portdb.porttree_root], pms_database): + if not isinstance(portage.portdb._pregen_auxdb[portage.portdb.repositories.mainRepoLocation()], pms_database): sys.exit(1) """),), @@ -106,13 +105,13 @@ class PortdbCacheTestCase(TestCase): (BASH_BINARY, "-c", "rm %s" % portage._shell_quote(layout_conf_path)), python_cmd + (textwrap.dedent(""" import os, sys, portage - if portage.portdb.porttree_root not in portage.portdb._pregen_auxdb: + if portage.portdb.repositories.mainRepoLocation() not in portage.portdb._pregen_auxdb: sys.exit(1) """),), python_cmd + (textwrap.dedent(""" import os, sys, portage from portage.cache.flat_hash import md5_database - if not isinstance(portage.portdb._pregen_auxdb[portage.portdb.porttree_root], md5_database): + if not isinstance(portage.portdb._pregen_auxdb[portage.portdb.repositories.mainRepoLocation()], md5_database): sys.exit(1) """),), ) diff --git a/pym/portage/tests/dbapi/__test__ b/pym/portage/tests/dep/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/dbapi/__test__ +++ b/pym/portage/tests/dep/__test__.py diff --git a/pym/portage/tests/dep/__test__ b/pym/portage/tests/ebuild/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/dep/__test__ +++ b/pym/portage/tests/ebuild/__test__.py diff --git a/pym/portage/tests/ebuild/test_config.py b/pym/portage/tests/ebuild/test_config.py index 08e0a5dcf..20aac519a 100644 --- a/pym/portage/tests/ebuild/test_config.py +++ b/pym/portage/tests/ebuild/test_config.py @@ -1,13 +1,20 @@ # Copyright 2010-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import unicode_literals + +import io +import tempfile + import portage -from portage import os +from portage import os, shutil, _encodings +from portage.const import USER_CONFIG_PATH from portage.dep import Atom from portage.package.ebuild.config import config from portage.package.ebuild._config.LicenseManager import LicenseManager from portage.tests import TestCase from portage.tests.resolver.ResolverPlayground import ResolverPlayground, ResolverPlaygroundTestCase +from portage.util import normalize_path class ConfigTestCase(TestCase): @@ -274,3 +281,65 @@ class ConfigTestCase(TestCase): self.assertEqual(test_case.test_success, True, test_case.fail_msg) finally: playground.cleanup() + + + def testSetCpv(self): + """ + Test the clone via constructor. + """ + + ebuilds = { + "dev-libs/A-1": {"IUSE": "static-libs"}, + "dev-libs/B-1": {"IUSE": "static-libs"}, + } + + env_files = { + "A" : ("USE=\"static-libs\"",) + } + + package_env = ( + "dev-libs/A A", + ) + + eprefix = normalize_path(tempfile.mkdtemp()) + playground = None + try: + user_config_dir = os.path.join(eprefix, USER_CONFIG_PATH) + os.makedirs(user_config_dir) + + with io.open(os.path.join(user_config_dir, "package.env"), + mode='w', encoding=_encodings['content']) as f: + for line in package_env: + f.write(line + "\n") + + env_dir = os.path.join(user_config_dir, "env") + os.makedirs(env_dir) + for k, v in env_files.items(): + with io.open(os.path.join(env_dir, k), mode='w', + encoding=_encodings['content']) as f: + for line in v: + f.write(line + "\n") + + playground = ResolverPlayground(eprefix=eprefix, ebuilds=ebuilds) + settings = config(clone=playground.settings) + + result = playground.run(["=dev-libs/A-1"]) + pkg, existing_node = result.depgraph._select_package( + playground.eroot, Atom("=dev-libs/A-1")) + settings.setcpv(pkg) + self.assertTrue("static-libs" in + settings["PORTAGE_USE"].split()) + + # Test bug #522362, where a USE=static-libs package.env + # setting leaked from one setcpv call to the next. + pkg, existing_node = result.depgraph._select_package( + playground.eroot, Atom("=dev-libs/B-1")) + settings.setcpv(pkg) + self.assertTrue("static-libs" not in + settings["PORTAGE_USE"].split()) + + finally: + if playground is None: + shutil.rmtree(eprefix) + else: + playground.cleanup() diff --git a/pym/portage/tests/ebuild/__test__ b/pym/portage/tests/emerge/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/ebuild/__test__ +++ b/pym/portage/tests/emerge/__test__.py diff --git a/pym/portage/tests/emerge/test_emerge_slot_abi.py b/pym/portage/tests/emerge/test_emerge_slot_abi.py index fd7ec0e6a..d1f2d9241 100644 --- a/pym/portage/tests/emerge/test_emerge_slot_abi.py +++ b/pym/portage/tests/emerge/test_emerge_slot_abi.py @@ -7,8 +7,7 @@ import sys import portage from portage import os from portage import _unicode_decode -from portage.const import (BASH_BINARY, PORTAGE_BIN_PATH, - PORTAGE_PYM_PATH, USER_CONFIG_PATH) +from portage.const import (BASH_BINARY, PORTAGE_PYM_PATH, USER_CONFIG_PATH) from portage.process import find_binary from portage.tests import TestCase from portage.tests.resolver.ResolverPlayground import ResolverPlayground @@ -70,9 +69,9 @@ class SlotAbiEmergeTestCase(TestCase): portage_python = portage._python_interpreter ebuild_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "ebuild")) + os.path.join(self.bindir, "ebuild")) emerge_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "emerge")) + os.path.join(self.bindir, "emerge")) test_ebuild = portdb.findname("dev-libs/dbus-glib-0.98") self.assertFalse(test_ebuild is None) diff --git a/pym/portage/tests/emerge/test_simple.py b/pym/portage/tests/emerge/test_simple.py index bf0af8bc8..0bb83ae4e 100644 --- a/pym/portage/tests/emerge/test_simple.py +++ b/pym/portage/tests/emerge/test_simple.py @@ -8,7 +8,7 @@ import portage from portage import os from portage import _unicode_decode from portage.const import (BASH_BINARY, PORTAGE_BASE_PATH, - PORTAGE_BIN_PATH, PORTAGE_PYM_PATH, USER_CONFIG_PATH) + PORTAGE_PYM_PATH, USER_CONFIG_PATH) from portage.process import find_binary from portage.tests import TestCase from portage.tests.resolver.ResolverPlayground import ResolverPlayground @@ -175,29 +175,29 @@ pkg_preinst() { portage_python = portage._python_interpreter dispatch_conf_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "dispatch-conf")) + os.path.join(self.sbindir, "dispatch-conf")) ebuild_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "ebuild")) + os.path.join(self.bindir, "ebuild")) egencache_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "egencache"), + os.path.join(self.bindir, "egencache"), "--repo", "test_repo", "--repositories-configuration", settings.repositories.config_string()) emerge_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "emerge")) + os.path.join(self.bindir, "emerge")) emaint_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "emaint")) + os.path.join(self.sbindir, "emaint")) env_update_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "env-update")) + os.path.join(self.sbindir, "env-update")) etc_update_cmd = (BASH_BINARY, - os.path.join(PORTAGE_BIN_PATH, "etc-update")) + os.path.join(self.sbindir, "etc-update")) fixpackages_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "fixpackages")) + os.path.join(self.sbindir, "fixpackages")) portageq_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "portageq")) + os.path.join(self.bindir, "portageq")) quickpkg_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "quickpkg")) + os.path.join(self.bindir, "quickpkg")) regenworld_cmd = (portage_python, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "regenworld")) + os.path.join(self.sbindir, "regenworld")) rm_binary = find_binary("rm") self.assertEqual(rm_binary is None, False, @@ -243,6 +243,12 @@ pkg_preinst() { emerge_cmd + ("--metadata",), rm_cmd + ("-rf", cachedir), emerge_cmd + ("--oneshot", "virtual/foo"), + lambda: self.assertFalse(os.path.exists( + os.path.join(pkgdir, "virtual", "foo-0.tbz2"))), + ({"FEATURES" : "unmerge-backup"},) + \ + emerge_cmd + ("--unmerge", "virtual/foo"), + lambda: self.assertTrue(os.path.exists( + os.path.join(pkgdir, "virtual", "foo-0.tbz2"))), emerge_cmd + ("--pretend", "dev-libs/A"), ebuild_cmd + (test_ebuild, "manifest", "clean", "package", "merge"), emerge_cmd + ("--pretend", "--tree", "--complete-graph", "dev-libs/A"), @@ -368,7 +374,7 @@ pkg_preinst() { for x in true_symlinks: os.symlink(true_binary, os.path.join(fake_bin, x)) for x in etc_symlinks: - os.symlink(os.path.join(PORTAGE_BASE_PATH, "cnf", x), + os.symlink(os.path.join(self.cnf_etc_path, x), os.path.join(eprefix, "etc", x)) with open(os.path.join(var_cache_edb, "counter"), 'wb') as f: f.write(b"100") @@ -395,6 +401,10 @@ move dev-util/git dev-vcs/git for args in test_commands: + if hasattr(args, '__call__'): + args() + continue + if isinstance(args[0], dict): local_env = env.copy() local_env.update(args[0]) diff --git a/pym/portage/tests/emerge/__test__ b/pym/portage/tests/env/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/emerge/__test__ +++ b/pym/portage/tests/env/__test__.py diff --git a/pym/portage/tests/env/__test__ b/pym/portage/tests/env/config/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/env/__test__ +++ b/pym/portage/tests/env/config/__test__.py diff --git a/pym/portage/tests/env/config/__test__ b/pym/portage/tests/glsa/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/env/config/__test__ +++ b/pym/portage/tests/glsa/__test__.py diff --git a/pym/portage/tests/glsa/test_security_set.py b/pym/portage/tests/glsa/test_security_set.py index edf567809..bf1f82b90 100644 --- a/pym/portage/tests/glsa/test_security_set.py +++ b/pym/portage/tests/glsa/test_security_set.py @@ -129,7 +129,8 @@ class SecuritySetTestCase(TestCase): try: portdb = playground.trees[playground.eroot]["porttree"].dbapi - glsa_dir = os.path.join(portdb.porttree_root, 'metadata', 'glsa') + glsa_dir = os.path.join( + portdb.repositories.mainRepoLocation(), 'metadata', 'glsa') portage.util.ensure_dirs(glsa_dir) for glsa in glsas: with io.open(os.path.join(glsa_dir, diff --git a/pym/portage/tests/glsa/__test__ b/pym/portage/tests/lafilefixer/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/glsa/__test__ +++ b/pym/portage/tests/lafilefixer/__test__.py diff --git a/pym/portage/tests/lafilefixer/__test__ b/pym/portage/tests/lazyimport/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/lafilefixer/__test__ +++ b/pym/portage/tests/lazyimport/__test__.py diff --git a/pym/portage/tests/lazyimport/__test__ b/pym/portage/tests/lint/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/lazyimport/__test__ +++ b/pym/portage/tests/lint/__test__.py diff --git a/pym/portage/tests/lint/test_compile_modules.py b/pym/portage/tests/lint/test_compile_modules.py index ce7e3fb90..4826cadb7 100644 --- a/pym/portage/tests/lint/test_compile_modules.py +++ b/pym/portage/tests/lint/test_compile_modules.py @@ -5,7 +5,7 @@ import errno import itertools import stat -from portage.const import PORTAGE_BIN_PATH, PORTAGE_PYM_PATH +from portage.const import PORTAGE_BIN_PATH, PORTAGE_PYM_PATH, PORTAGE_PYM_PACKAGES from portage.tests import TestCase from portage import os from portage import _encodings @@ -14,9 +14,11 @@ from portage import _unicode_decode, _unicode_encode class CompileModulesTestCase(TestCase): def testCompileModules(self): - for parent, _dirs, files in itertools.chain( - os.walk(PORTAGE_BIN_PATH), - os.walk(PORTAGE_PYM_PATH)): + iters = [os.walk(os.path.join(PORTAGE_PYM_PATH, x)) + for x in PORTAGE_PYM_PACKAGES] + iters.append(os.walk(PORTAGE_BIN_PATH)) + + for parent, _dirs, files in itertools.chain(*iters): parent = _unicode_decode(parent, encoding=_encodings['fs'], errors='strict') for x in files: diff --git a/pym/portage/tests/lint/test_import_modules.py b/pym/portage/tests/lint/test_import_modules.py index 34261f464..fcdcb3b33 100644 --- a/pym/portage/tests/lint/test_import_modules.py +++ b/pym/portage/tests/lint/test_import_modules.py @@ -1,7 +1,9 @@ # Copyright 2011-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from portage.const import PORTAGE_PYM_PATH +from itertools import chain + +from portage.const import PORTAGE_PYM_PATH, PORTAGE_PYM_PACKAGES from portage.tests import TestCase from portage import os from portage import _encodings @@ -13,7 +15,9 @@ class ImportModulesTestCase(TestCase): expected_failures = frozenset(( )) - for mod in self._iter_modules(PORTAGE_PYM_PATH): + iters = (self._iter_modules(os.path.join(PORTAGE_PYM_PATH, x)) + for x in PORTAGE_PYM_PACKAGES) + for mod in chain(*iters): try: __import__(mod) except ImportError as e: diff --git a/pym/portage/tests/lint/__test__ b/pym/portage/tests/locks/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/lint/__test__ +++ b/pym/portage/tests/locks/__test__.py diff --git a/pym/portage/tests/locks/__test__ b/pym/portage/tests/news/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/locks/__test__ +++ b/pym/portage/tests/news/__test__.py diff --git a/pym/portage/tests/news/__test__ b/pym/portage/tests/process/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/news/__test__ +++ b/pym/portage/tests/process/__test__.py diff --git a/pym/portage/tests/process/__test__ b/pym/portage/tests/repoman/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/process/__test__ +++ b/pym/portage/tests/repoman/__test__.py diff --git a/pym/portage/tests/repoman/test_simple.py b/pym/portage/tests/repoman/test_simple.py index 69eb36de8..5dbb7676f 100644 --- a/pym/portage/tests/repoman/test_simple.py +++ b/pym/portage/tests/repoman/test_simple.py @@ -9,7 +9,7 @@ import portage from portage import os from portage import shutil from portage import _unicode_decode -from portage.const import PORTAGE_BASE_PATH, PORTAGE_BIN_PATH, PORTAGE_PYM_PATH +from portage.const import PORTAGE_BASE_PATH, PORTAGE_PYM_PATH from portage.process import find_binary from portage.tests import TestCase from portage.tests.resolver.ResolverPlayground import ResolverPlayground @@ -171,7 +171,7 @@ class SimpleRepomanTestCase(TestCase): license_dir = os.path.join(test_repo_location, "licenses") repoman_cmd = (portage._python_interpreter, "-b", "-Wd", - os.path.join(PORTAGE_BIN_PATH, "repoman")) + os.path.join(self.bindir, "repoman")) git_binary = find_binary("git") git_cmd = (git_binary,) @@ -274,7 +274,9 @@ class SimpleRepomanTestCase(TestCase): os.symlink(test_repo_location, test_repo_symlink) # repoman checks metadata.dtd for recent CTIME, so copy the file in # order to ensure that the CTIME is current - shutil.copyfile(metadata_dtd, os.path.join(distdir, "metadata.dtd")) + # NOTE: if we don't have the file around, let repoman try to fetch it. + if os.path.exists(metadata_dtd): + shutil.copyfile(metadata_dtd, os.path.join(distdir, "metadata.dtd")) if debug: # The subprocess inherits both stdout and stderr, for diff --git a/pym/portage/tests/resolver/ResolverPlayground.py b/pym/portage/tests/resolver/ResolverPlayground.py index 077e27159..2d162514c 100644 --- a/pym/portage/tests/resolver/ResolverPlayground.py +++ b/pym/portage/tests/resolver/ResolverPlayground.py @@ -15,6 +15,7 @@ from portage.package.ebuild.config import config from portage.package.ebuild.digestgen import digestgen from portage._sets import load_default_config from portage._sets.base import InternalPackageSet +from portage.tests import cnf_path from portage.util import ensure_dirs, normalize_path from portage.versions import catsplit @@ -58,15 +59,19 @@ class ResolverPlayground(object): def __init__(self, ebuilds={}, binpkgs={}, installed={}, profile={}, repo_configs={}, \ user_config={}, sets={}, world=[], world_sets=[], distfiles={}, - targetroot=False, debug=False): + eprefix=None, targetroot=False, debug=False): """ ebuilds: cpv -> metadata mapping simulating available ebuilds. installed: cpv -> metadata mapping simulating installed packages. If a metadata key is missing, it gets a default value. profile: settings defined by the profile. """ + self.debug = debug - self.eprefix = normalize_path(tempfile.mkdtemp()) + if eprefix is None: + self.eprefix = normalize_path(tempfile.mkdtemp()) + else: + self.eprefix = normalize_path(eprefix) portage.const.EPREFIX = self.eprefix.rstrip(os.sep) self.eroot = self.eprefix + os.sep @@ -414,7 +419,7 @@ class ResolverPlayground(object): make_globals_path = os.path.join(self.eroot, GLOBAL_CONFIG_PATH.lstrip(os.sep), "make.globals") ensure_dirs(os.path.dirname(make_globals_path)) - os.symlink(os.path.join(PORTAGE_BASE_PATH, "cnf", "make.globals"), + os.symlink(os.path.join(cnf_path, "make.globals"), make_globals_path) #Create /usr/share/portage/config/sets/portage.conf @@ -425,8 +430,8 @@ class ResolverPlayground(object): except os.error: pass - provided_sets_portage_conf = \ - os.path.join(PORTAGE_BASE_PATH, "cnf/sets/portage.conf") + provided_sets_portage_conf = ( + os.path.join(cnf_path, "sets", "portage.conf")) os.symlink(provided_sets_portage_conf, os.path.join(default_sets_conf_dir, "portage.conf")) set_config_dir = os.path.join(user_config_dir, "sets") @@ -544,6 +549,7 @@ class ResolverPlaygroundTestCase(object): self.all_permutations = kwargs.pop("all_permutations", False) self.ignore_mergelist_order = kwargs.pop("ignore_mergelist_order", False) self.ambiguous_merge_order = kwargs.pop("ambiguous_merge_order", False) + self.ambiguous_slot_collision_solutions = kwargs.pop("ambiguous_slot_collision_solutions", False) self.check_repo_names = kwargs.pop("check_repo_names", False) self.merge_order_assertions = kwargs.pop("merge_order_assertions", False) @@ -659,9 +665,22 @@ class ResolverPlaygroundTestCase(object): str((node1, node2))) + \ ", got: " + str(got)) - elif key in ("unstable_keywords", "needed_p_mask_changes") and expected is not None: + elif key == "slot_collision_solutions" and \ + self.ambiguous_slot_collision_solutions: + # Tests that use all_permutations can have multiple + # outcomes here. + for x in expected: + if x == got: + expected = x + break + elif key in ("unstable_keywords", "needed_p_mask_changes", + "unsatisfied_deps", "required_use_unsatisfied") and \ + expected is not None: expected = set(expected) + elif key == "forced_rebuilds" and expected is not None: + expected = dict((k, set(v)) for k, v in expected.items()) + if got != expected: fail_msgs.append("atoms: (" + ", ".join(result.atoms) + "), key: " + \ key + ", expected: " + str(expected) + ", got: " + str(got)) @@ -674,10 +693,15 @@ class ResolverPlaygroundTestCase(object): class ResolverPlaygroundResult(object): checks = ( - "success", "mergelist", "use_changes", "license_changes", "unstable_keywords", "slot_collision_solutions", + "success", "mergelist", "use_changes", "license_changes", + "unstable_keywords", "slot_collision_solutions", "circular_dependency_solutions", "needed_p_mask_changes", + "unsatisfied_deps", "forced_rebuilds", "required_use_unsatisfied" ) optional_checks = ( + "forced_rebuilds", + "required_use_unsatisfied", + "unsatisfied_deps" ) def __init__(self, atoms, success, mydepgraph, favorites): @@ -692,6 +716,9 @@ class ResolverPlaygroundResult(object): self.needed_p_mask_changes = None self.slot_collision_solutions = None self.circular_dependency_solutions = None + self.unsatisfied_deps = frozenset() + self.forced_rebuilds = None + self.required_use_unsatisfied = None if self.depgraph._dynamic_config._serialized_tasks_cache is not None: self.mergelist = [] @@ -751,6 +778,24 @@ class ResolverPlaygroundResult(object): sol = handler.solutions self.circular_dependency_solutions = dict(zip([x.cpv for x in sol.keys()], sol.values())) + if self.depgraph._dynamic_config._unsatisfied_deps_for_display: + self.unsatisfied_deps = set(dep_info[0][1] + for dep_info in self.depgraph._dynamic_config._unsatisfied_deps_for_display) + + if self.depgraph._forced_rebuilds: + self.forced_rebuilds = dict( + (child.cpv, set(parent.cpv for parent in parents)) + for child_dict in self.depgraph._forced_rebuilds.values() + for child, parents in child_dict.items()) + + required_use_unsatisfied = [] + for pargs, kwargs in \ + self.depgraph._dynamic_config._unsatisfied_deps_for_display: + if "show_req_use" in kwargs: + required_use_unsatisfied.append(pargs[1]) + if required_use_unsatisfied: + self.required_use_unsatisfied = set(required_use_unsatisfied) + class ResolverPlaygroundDepcleanResult(object): checks = ( diff --git a/pym/portage/tests/repoman/__test__ b/pym/portage/tests/resolver/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/repoman/__test__ +++ b/pym/portage/tests/resolver/__test__.py diff --git a/pym/portage/tests/resolver/test_autounmask_use_breakage.py b/pym/portage/tests/resolver/test_autounmask_use_breakage.py new file mode 100644 index 000000000..3654aa6a3 --- /dev/null +++ b/pym/portage/tests/resolver/test_autounmask_use_breakage.py @@ -0,0 +1,63 @@ +# Copyright 2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import (ResolverPlayground, + ResolverPlaygroundTestCase) + +class AutounmaskUseBreakageTestCase(TestCase): + + def testAutounmaskUseBreakage(self): + + ebuilds = { + + "app-misc/A-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/D[-foo]", + }, + + "app-misc/B-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/D[foo]" + }, + + "app-misc/C-0" : { + "EAPI": "5", + "RDEPEND": ">=app-misc/D-1" + }, + + "app-misc/D-0" : { + "EAPI": "5", + "IUSE": "foo" + }, + + "app-misc/D-1" : { + "EAPI": "5", + "IUSE": "bar" + }, + + } + + test_cases = ( + + # Bug 510270 + # _solve_non_slot_operator_slot_conflicts throws + # IndexError: tuple index out of range + # due to autounmask USE breakage. + ResolverPlaygroundTestCase( + ["app-misc/C", "app-misc/B", "app-misc/A"], + all_permutations = True, + success = False, + ambiguous_slot_collision_solutions = True, + slot_collision_solutions = [None, []] + ), + + ) + + playground = ResolverPlayground(ebuilds=ebuilds, debug=False) + try: + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, True, test_case.fail_msg) + finally: + playground.cleanup() diff --git a/pym/portage/tests/resolver/test_or_choices.py b/pym/portage/tests/resolver/test_or_choices.py index 90e681408..d9d14f058 100644 --- a/pym/portage/tests/resolver/test_or_choices.py +++ b/pym/portage/tests/resolver/test_or_choices.py @@ -132,3 +132,76 @@ class OrChoicesTestCase(TestCase): self.assertEqual(test_case.test_success, True, test_case.fail_msg) finally: playground.cleanup() + + + def testInitiallyUnsatisfied(self): + + ebuilds = { + + "app-misc/A-1" : { + "EAPI": "5", + "SLOT": "0/1" + }, + + "app-misc/A-2" : { + "EAPI": "5", + "SLOT": "0/2" + }, + + "app-misc/B-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/A:=" + }, + + "app-misc/C-0" : { + "EAPI": "5", + "RDEPEND": "|| ( app-misc/X <app-misc/A-2 )" + }, + + } + + installed = { + + "app-misc/A-1" : { + "EAPI": "5", + "SLOT": "0/1" + }, + + "app-misc/B-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/A:0/1=" + }, + + "app-misc/C-0" : { + "EAPI": "5", + "RDEPEND": "|| ( app-misc/X <app-misc/A-2 )" + }, + + } + + world = ["app-misc/B", "app-misc/C"] + + test_cases = ( + + # Test bug #522652, where the unsatisfiable app-misc/X + # atom is selected, and the dependency is placed into + # _initially_unsatisfied_deps where it is ignored, causing + # upgrade to app-misc/A-2 (breaking a dependency of + # app-misc/C-0). + ResolverPlaygroundTestCase( + ["app-misc/A"], + options = {}, + success = True, + mergelist = ['app-misc/A-1'] + ), + + ) + + playground = ResolverPlayground(ebuilds=ebuilds, + installed=installed, world=world, debug=False) + try: + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, True, test_case.fail_msg) + finally: + playground.cleanup() diff --git a/pym/portage/tests/resolver/test_slot_conflict_force_rebuild.py b/pym/portage/tests/resolver/test_slot_conflict_force_rebuild.py new file mode 100644 index 000000000..4170bfd9d --- /dev/null +++ b/pym/portage/tests/resolver/test_slot_conflict_force_rebuild.py @@ -0,0 +1,84 @@ +# Copyright 2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import (ResolverPlayground, + ResolverPlaygroundTestCase) + +class SlotConflictForceRebuildTestCase(TestCase): + + def testSlotConflictForceRebuild(self): + + ebuilds = { + + "app-misc/A-1" : { + "EAPI": "5", + "SLOT": "0/1" + }, + + "app-misc/A-2" : { + "EAPI": "5", + "SLOT": "0/2" + }, + + "app-misc/B-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/A:=" + }, + + "app-misc/C-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/A" + }, + + } + + installed = { + + "app-misc/A-1" : { + "EAPI": "5", + "SLOT": "0/1" + }, + + "app-misc/B-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/A:0/1=" + }, + + "app-misc/C-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/A:0/1=" + }, + + } + + world = ["app-misc/B", "app-misc/C"] + + test_cases = ( + + # Test bug #521990, where forced_rebuilds omits ebuilds that + # had have had their slot operator atoms removed from the + # ebuilds, even though the corresponding installed + # instances had really forced rebuilds due to being built + # with slot-operators in their deps. + ResolverPlaygroundTestCase( + ["app-misc/A"], + options = {}, + success = True, + ambiguous_merge_order = True, + mergelist = ['app-misc/A-2', ('app-misc/B-0', 'app-misc/C-0')], + forced_rebuilds = { + 'app-misc/A-2': ['app-misc/B-0', 'app-misc/C-0'] + } + ), + + ) + + playground = ResolverPlayground(ebuilds=ebuilds, + installed=installed, world=world, debug=False) + try: + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, True, test_case.fail_msg) + finally: + playground.cleanup() diff --git a/pym/portage/tests/resolver/test_slot_conflict_unsatisfied_deep_deps.py b/pym/portage/tests/resolver/test_slot_conflict_unsatisfied_deep_deps.py new file mode 100644 index 000000000..13f7e67e3 --- /dev/null +++ b/pym/portage/tests/resolver/test_slot_conflict_unsatisfied_deep_deps.py @@ -0,0 +1,115 @@ +# Copyright 2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import ResolverPlayground, ResolverPlaygroundTestCase + +class SlotConflictUnsatisfiedDeepDepsTestCase(TestCase): + + def testSlotConflictUnsatisfiedDeepDeps(self): + + ebuilds = { + "dev-libs/A-1": { }, + "dev-libs/A-2": { "KEYWORDS": "~x86" }, + "dev-libs/B-1": { "DEPEND": "dev-libs/A" }, + "dev-libs/C-1": { "DEPEND": ">=dev-libs/A-2" }, + "dev-libs/D-1": { "DEPEND": "dev-libs/A" }, + } + + installed = { + "dev-libs/broken-1": { + "RDEPEND": "dev-libs/A dev-libs/initially-unsatisfied" + }, + } + + world = ( + "dev-libs/A", + "dev-libs/B", + "dev-libs/C", + "dev-libs/D", + "dev-libs/broken" + ) + + test_cases = ( + # Test bug #520950, where unsatisfied deps of installed + # packages are supposed to be ignored when they are beyond + # the depth requested by the user. + ResolverPlaygroundTestCase( + ["dev-libs/B", "dev-libs/C", "dev-libs/D"], + all_permutations=True, + options={ + "--autounmask": "y", + "--complete-graph": True + }, + mergelist=["dev-libs/A-2", "dev-libs/B-1", "dev-libs/C-1", "dev-libs/D-1"], + ignore_mergelist_order=True, + unstable_keywords=["dev-libs/A-2"], + unsatisfied_deps=[], + success=False), + + ResolverPlaygroundTestCase( + ["@world"], + options={ + "--autounmask": "y", + "--complete-graph": True + }, + mergelist=["dev-libs/A-2", "dev-libs/B-1", "dev-libs/C-1", "dev-libs/D-1"], + ignore_mergelist_order=True, + unstable_keywords=["dev-libs/A-2"], + unsatisfied_deps=["dev-libs/broken"], + success=False), + + # Test --selective with --deep = 0 + ResolverPlaygroundTestCase( + ["@world"], + options={ + "--autounmask": "y", + "--complete-graph": True, + "--selective": True, + "--deep": 0 + }, + mergelist=["dev-libs/A-2", "dev-libs/B-1", "dev-libs/C-1", "dev-libs/D-1"], + ignore_mergelist_order=True, + unstable_keywords=["dev-libs/A-2"], + unsatisfied_deps=[], + success=False), + + # Test --deep = 1 + ResolverPlaygroundTestCase( + ["@world"], + options={ + "--autounmask": "y", + "--complete-graph": True, + "--selective": True, + "--deep": 1 + }, + mergelist=["dev-libs/A-2", "dev-libs/B-1", "dev-libs/C-1", "dev-libs/D-1"], + ignore_mergelist_order=True, + unstable_keywords=["dev-libs/A-2"], + unsatisfied_deps=["dev-libs/initially-unsatisfied"], + success=False), + + # Test --deep = True + ResolverPlaygroundTestCase( + ["@world"], + options={ + "--autounmask": "y", + "--complete-graph": True, + "--selective": True, + "--deep": True + }, + mergelist=["dev-libs/A-2", "dev-libs/B-1", "dev-libs/C-1", "dev-libs/D-1"], + ignore_mergelist_order=True, + unstable_keywords=["dev-libs/A-2"], + unsatisfied_deps=["dev-libs/initially-unsatisfied"], + success=False), + ) + + playground = ResolverPlayground(ebuilds=ebuilds, installed=installed, + world=world, debug=False) + try: + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, True, test_case.fail_msg) + finally: + playground.cleanup() diff --git a/pym/portage/tests/resolver/test_slot_operator_rebuild.py b/pym/portage/tests/resolver/test_slot_operator_rebuild.py new file mode 100644 index 000000000..42512aad8 --- /dev/null +++ b/pym/portage/tests/resolver/test_slot_operator_rebuild.py @@ -0,0 +1,80 @@ +# Copyright 2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import (ResolverPlayground, + ResolverPlaygroundTestCase) + +class SlotOperatorRebuildTestCase(TestCase): + + def testSlotOperatorRebuild(self): + + ebuilds = { + + "app-misc/A-1" : { + "EAPI": "5", + "SLOT": "0/1" + }, + + "app-misc/A-2" : { + "EAPI": "5", + "SLOT": "0/2" + }, + + "app-misc/B-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/A:=" + }, + + "app-misc/C-0" : { + "EAPI": "5", + "RDEPEND": "|| ( app-misc/X app-misc/A:= )" + }, + + } + + installed = { + + "app-misc/A-1" : { + "EAPI": "5", + "SLOT": "0/1" + }, + + "app-misc/B-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/A:0/1=" + }, + + "app-misc/C-0" : { + "EAPI": "5", + "RDEPEND": "|| ( app-misc/X app-misc/A:0/1= )" + }, + + } + + world = ["app-misc/B", "app-misc/C"] + + test_cases = ( + + # Test bug #522652, where the unsatisfiable app-misc/X + # atom is selected, and the dependency is placed into + # _initially_unsatisfied_deps where it is ignored, causing + # the app-misc/C-0 rebuild to be missed. + ResolverPlaygroundTestCase( + ["app-misc/A"], + options = {"--dynamic-deps": "n"}, + success = True, + ambiguous_merge_order = True, + mergelist = ['app-misc/A-2', ('app-misc/B-0', 'app-misc/C-0')] + ), + + ) + + playground = ResolverPlayground(ebuilds=ebuilds, + installed=installed, world=world, debug=False) + try: + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, True, test_case.fail_msg) + finally: + playground.cleanup() diff --git a/pym/portage/tests/resolver/test_slot_operator_required_use.py b/pym/portage/tests/resolver/test_slot_operator_required_use.py new file mode 100644 index 000000000..9cc6dbad4 --- /dev/null +++ b/pym/portage/tests/resolver/test_slot_operator_required_use.py @@ -0,0 +1,72 @@ +# Copyright 2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import (ResolverPlayground, + ResolverPlaygroundTestCase) + +class SlotOperatorRequiredUseTestCase(TestCase): + + def testSlotOperatorRequiredUse(self): + + ebuilds = { + + "app-misc/A-1" : { + "EAPI": "5", + "SLOT": "0/1" + }, + + "app-misc/A-2" : { + "EAPI": "5", + "SLOT": "0/2" + }, + + "app-misc/B-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/A:=", + "IUSE": "x y", + "REQUIRED_USE": "|| ( x y )" + }, + + } + + installed = { + + "app-misc/A-1" : { + "EAPI": "5", + "SLOT": "0/1" + }, + + "app-misc/B-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/A:0/1=", + "IUSE": "x y", + "USE": "x" + }, + + } + + world = ["app-misc/B"] + + test_cases = ( + + # bug 523048 + # Ensure that unsatisfied REQUIRED_USE is reported when + # it blocks necessary slot-operator rebuilds. + ResolverPlaygroundTestCase( + ["app-misc/A"], + success = False, + required_use_unsatisfied = ['app-misc/B:0'] + ), + + ) + + playground = ResolverPlayground(ebuilds=ebuilds, + installed=installed, world=world, debug=False) + try: + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, True, + test_case.fail_msg) + finally: + playground.cleanup() diff --git a/pym/portage/tests/resolver/test_solve_non_slot_operator_slot_conflicts.py b/pym/portage/tests/resolver/test_solve_non_slot_operator_slot_conflicts.py new file mode 100644 index 000000000..c6024f404 --- /dev/null +++ b/pym/portage/tests/resolver/test_solve_non_slot_operator_slot_conflicts.py @@ -0,0 +1,75 @@ +# Copyright 2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import (ResolverPlayground, + ResolverPlaygroundTestCase) + +class SolveNonSlotOperatorSlotConflictsTestCase(TestCase): + + def testSolveNonSlotOperatorSlotConflicts(self): + + ebuilds = { + + "app-misc/A-1" : { + "EAPI": "5", + "SLOT": "0/1", + "PDEPEND": "app-misc/B" + }, + + "app-misc/A-2" : { + "EAPI": "5", + "SLOT": "0/2", + "PDEPEND": "app-misc/B" + }, + + "app-misc/B-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/A:=" + }, + + } + + installed = { + + "app-misc/A-1" : { + "EAPI": "5", + "SLOT": "0/1", + "PDEPEND": "app-misc/B" + }, + + "app-misc/B-0" : { + "EAPI": "5", + "RDEPEND": "app-misc/A:0/1=" + }, + + } + + world = ["app-misc/A"] + + test_cases = ( + + # bug 522084 + # In this case, _solve_non_slot_operator_slot_conflicts + # removed both versions of app-misc/A from the graph, since + # they didn't have any non-conflict parents (except for + # @selected which matched both instances). The result was + # a missed update. + ResolverPlaygroundTestCase( + ["@world"], + options = {"--update": True, "--deep": True}, + success = True, + mergelist = ['app-misc/A-2', 'app-misc/B-0'] + ), + + ) + + playground = ResolverPlayground(ebuilds=ebuilds, + installed=installed, world=world, debug=False) + try: + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, True, + test_case.fail_msg) + finally: + playground.cleanup() diff --git a/pym/portage/tests/runTests b/pym/portage/tests/runTests.py index a80668b8d..a80668b8d 100755 --- a/pym/portage/tests/runTests +++ b/pym/portage/tests/runTests.py diff --git a/pym/portage/tests/resolver/__test__ b/pym/portage/tests/sets/base/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/resolver/__test__ +++ b/pym/portage/tests/sets/base/__test__.py diff --git a/pym/portage/tests/sets/base/__test__ b/pym/portage/tests/sets/files/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/sets/base/__test__ +++ b/pym/portage/tests/sets/files/__test__.py diff --git a/pym/portage/tests/sets/files/__test__ b/pym/portage/tests/sets/shell/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/sets/files/__test__ +++ b/pym/portage/tests/sets/shell/__test__.py diff --git a/pym/portage/tests/sets/shell/__test__ b/pym/portage/tests/unicode/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/sets/shell/__test__ +++ b/pym/portage/tests/unicode/__test__.py diff --git a/pym/portage/tests/unicode/__test__ b/pym/portage/tests/update/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/unicode/__test__ +++ b/pym/portage/tests/update/__test__.py diff --git a/pym/portage/tests/update/__test__ b/pym/portage/tests/util/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/update/__test__ +++ b/pym/portage/tests/util/__test__.py diff --git a/pym/portage/tests/util/test_getconfig.py b/pym/portage/tests/util/test_getconfig.py index e5fd60f6d..b72bd6a02 100644 --- a/pym/portage/tests/util/test_getconfig.py +++ b/pym/portage/tests/util/test_getconfig.py @@ -26,9 +26,7 @@ class GetConfigTestCase(TestCase): } def testGetConfig(self): - - make_globals_file = os.path.join(PORTAGE_BASE_PATH, - 'cnf', 'make.globals') + make_globals_file = os.path.join(self.cnf_path, "make.globals") d = getconfig(make_globals_file) for k, v in self._cases.items(): self.assertEqual(d[k], v) diff --git a/pym/portage/tests/util/__test__ b/pym/portage/tests/versions/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/util/__test__ +++ b/pym/portage/tests/versions/__test__.py diff --git a/pym/portage/tests/xpak/__test__ b/pym/portage/tests/xpak/__test__ deleted file mode 100644 index e69de29bb..000000000 --- a/pym/portage/tests/xpak/__test__ +++ /dev/null diff --git a/pym/portage/tests/versions/__test__ b/pym/portage/tests/xpak/__test__.py index e69de29bb..e69de29bb 100644 --- a/pym/portage/tests/versions/__test__ +++ b/pym/portage/tests/xpak/__test__.py diff --git a/pym/portage/util/__init__.py b/pym/portage/util/__init__.py index 9e2d1e40e..d48b47a40 100644 --- a/pym/portage/util/__init__.py +++ b/pym/portage/util/__init__.py @@ -43,7 +43,8 @@ from portage import _unicode_encode from portage import _unicode_decode from portage.const import VCS_DIRS from portage.exception import InvalidAtom, PortageException, FileNotFound, \ - OperationNotPermitted, ParseError, PermissionDenied, ReadOnlyFileSystem + IsADirectory, OperationNotPermitted, ParseError, PermissionDenied, \ + ReadOnlyFileSystem from portage.localization import _ from portage.proxy.objectproxy import ObjectProxy from portage.cache.mappings import UserDict diff --git a/pym/portage/util/_eventloop/EventLoop.py b/pym/portage/util/_eventloop/EventLoop.py index 9ffcc74d9..809540023 100644 --- a/pym/portage/util/_eventloop/EventLoop.py +++ b/pym/portage/util/_eventloop/EventLoop.py @@ -1,6 +1,8 @@ -# Copyright 1999-2013 Gentoo Foundation +# Copyright 1999-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import division + import errno import logging import os @@ -211,7 +213,7 @@ class EventLoop(object): if timeout is None: wait_timeout = None else: - wait_timeout = float(timeout) / 1000 + wait_timeout = timeout / 1000 # NOTE: In order to avoid a possible infinite wait when # wait_timeout is None, the previous _run_timeouts() # call must have returned False *with* _thread_condition @@ -657,6 +659,6 @@ class _epoll_adapter(object): if timeout is None or timeout < 0: timeout = -1 elif timeout != 0: - timeout = float(timeout) / 1000 + timeout = timeout / 1000 return self._epoll_obj.poll(timeout) diff --git a/pym/portage/util/_eventloop/PollSelectAdapter.py b/pym/portage/util/_eventloop/PollSelectAdapter.py index 244788c57..32b404b67 100644 --- a/pym/portage/util/_eventloop/PollSelectAdapter.py +++ b/pym/portage/util/_eventloop/PollSelectAdapter.py @@ -1,6 +1,8 @@ -# Copyright 1999-2012 Gentoo Foundation +# Copyright 1999-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import division + from .PollConstants import PollConstants import select @@ -64,7 +66,7 @@ class PollSelectAdapter(object): if timeout is not None and timeout < 0: timeout = None if timeout is not None: - select_args.append(float(timeout) / 1000) + select_args.append(timeout / 1000) select_events = select.select(*select_args) poll_events = [] diff --git a/pym/repoman/checks.py b/pym/repoman/checks.py index abb954569..5f3764832 100644 --- a/pym/repoman/checks.py +++ b/pym/repoman/checks.py @@ -395,11 +395,8 @@ class InheritDeprecated(LineCheck): def new(self, pkg): self._errors = [] - self._indirect_deprecated = set(eclass for eclass in \ - self.deprecated_classes if eclass in pkg.inherited) def check(self, num, line): - direct_inherits = None m = self._inherit_re.match(line) if m is not None: @@ -415,11 +412,9 @@ class InheritDeprecated(LineCheck): if replacement is None: pass elif replacement is False: - self._indirect_deprecated.discard(eclass) self._errors.append("please migrate from " + \ "'%s' (no replacement) on line: %d" % (eclass, num + 1)) else: - self._indirect_deprecated.discard(eclass) self._errors.append("please migrate from " + \ "'%s' to '%s' on line: %d" % \ (eclass, replacement, num + 1)) @@ -429,17 +424,6 @@ class InheritDeprecated(LineCheck): yield error del self._errors - for eclass in self._indirect_deprecated: - replacement = self.deprecated_classes[eclass] - if replacement is False: - yield "please migrate from indirect " + \ - "inherit of '%s' (no replacement)" % (eclass,) - else: - yield "please migrate from indirect " + \ - "inherit of '%s' to '%s'" % \ - (eclass, replacement) - del self._indirect_deprecated - class InheritEclass(LineCheck): """ Base class for checking for missing inherits, as well as excess inherits. diff --git a/runtests.sh b/runtests.sh index 9b7ea4236..2ff620635 100755 --- a/runtests.sh +++ b/runtests.sh @@ -4,7 +4,7 @@ # These are the versions we care about. The rest are just "nice to have". PYTHON_SUPPORTED_VERSIONS="2.6 2.7 3.2 3.3 3.4" -PYTHON_VERSIONS="2.6 2.7 2.7-pypy-1.8 2.7-pypy-1.9 2.7-pypy-2.0 3.1 3.2 3.3 3.4 3.5" +PYTHON_VERSIONS="2.6 2.7 pypy 3.1 3.2 3.3 3.4 3.5" # has to be run from portage root dir cd "${0%/*}" || exit 1 @@ -60,14 +60,14 @@ exit_status="0" found_versions=() status_array=() for version in ${PYTHON_VERSIONS}; do - if [[ $version =~ ^([[:digit:]]+\.[[:digit:]]+)-pypy-([[:digit:]]+\.[[:digit:]]+)$ ]] ; then - executable=${eprefix}/usr/bin/pypy-c${BASH_REMATCH[2]} + if [[ $version = 'pypy' ]] ; then + executable=${eprefix}/usr/bin/pypy else executable=${eprefix}/usr/bin/python${version} fi if [[ -x "${executable}" ]]; then echo -e "${GOOD}Testing with Python ${version}...${NORMAL}" - "${executable}" -b -Wd pym/portage/tests/runTests "$@" + "${executable}" -b -Wd pym/portage/tests/runTests.py "$@" status=$? status_array[${#status_array[@]}]=${status} found_versions[${#found_versions[@]}]=${version} diff --git a/setup.py b/setup.py new file mode 100755 index 000000000..22d779056 --- /dev/null +++ b/setup.py @@ -0,0 +1,652 @@ +#!/usr/bin/env python +# Copyright 1998-2014 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from distutils.core import setup, Command +from distutils.command.build import build +from distutils.command.build_scripts import build_scripts +from distutils.command.clean import clean +from distutils.command.install import install +from distutils.command.install_data import install_data +from distutils.command.install_lib import install_lib +from distutils.command.install_scripts import install_scripts +from distutils.command.sdist import sdist +from distutils.dep_util import newer +from distutils.dir_util import mkpath, remove_tree +from distutils.util import change_root, subst_vars + +import codecs +import collections +import glob +import os +import os.path +import re +import subprocess +import sys + + +# TODO: +# - smarter rebuilds of docs w/ 'install_docbook' and 'install_epydoc'. + +x_scripts = { + 'bin': [ + 'bin/archive-conf', 'bin/dispatch-conf', 'bin/ebuild', 'bin/egencache', + 'bin/emaint', 'bin/emerge', 'bin/emerge-webrsync', 'bin/emirrordist', + 'bin/env-update', 'bin/etc-update', 'bin/fixpackages', 'bin/portageq', + 'bin/quickpkg', 'bin/regenworld', 'bin/repoman', + ], +} + + +class x_build(build): + """ Build command with extra build_man call. """ + + def run(self): + build.run(self) + self.run_command('build_man') + + +class build_man(Command): + """ Perform substitutions in manpages. """ + + user_options = [ + ] + + def initialize_options(self): + self.build_base = None + + def finalize_options(self): + self.set_undefined_options('build', + ('build_base', 'build_base')) + + def run(self): + for d, files in self.distribution.data_files: + if not d.startswith('$mandir/'): + continue + + for source in files: + target = os.path.join(self.build_base, source) + mkpath(os.path.dirname(target)) + + if not newer(source, target) and not newer(__file__, target): + continue + + print('copying and updating %s -> %s' % ( + source, target)) + + with codecs.open(source, 'r', 'utf8') as f: + data = f.readlines() + data[0] = data[0].replace('VERSION', + self.distribution.get_version()) + with codecs.open(target, 'w', 'utf8') as f: + f.writelines(data) + + +class docbook(Command): + """ Build docs using docbook. """ + + user_options = [ + ('doc-formats=', None, 'Documentation formats to build (all xmlto formats for docbook are allowed, comma-separated'), + ] + + def initialize_options(self): + self.doc_formats = 'xhtml,xhtml-nochunks' + + def finalize_options(self): + self.doc_formats = self.doc_formats.replace(',', ' ').split() + + def run(self): + if not os.path.isdir('doc/fragment'): + mkpath('doc/fragment') + + with open('doc/fragment/date', 'w'): + pass + with open('doc/fragment/version', 'w') as f: + f.write('<releaseinfo>%s</releaseinfo>' % self.distribution.get_version()) + + for f in self.doc_formats: + print('Building docs in %s format...' % f) + subprocess.check_call(['xmlto', '-o', 'doc', + '-m', 'doc/custom.xsl', f, 'doc/portage.docbook']) + + +class epydoc(Command): + """ Build API docs using epydoc. """ + + user_options = [ + ] + + def initialize_options(self): + self.build_lib = None + + def finalize_options(self): + self.set_undefined_options('build_py', ('build_lib', 'build_lib')) + + def run(self): + self.run_command('build_py') + + print('Building API documentation...') + + process_env = os.environ.copy() + pythonpath = self.build_lib + try: + pythonpath += ':' + process_env['PYTHONPATH'] + except KeyError: + pass + process_env['PYTHONPATH'] = pythonpath + + subprocess.check_call(['epydoc', '-o', 'epydoc', + '--name', self.distribution.get_name(), + '--url', self.distribution.get_url(), + '-qq', '--no-frames', '--show-imports', + '--exclude', 'portage.tests', + '_emerge', 'portage', 'repoman'], + env = process_env) + os.remove('epydoc/api-objects.txt') + + +class install_docbook(install_data): + """ install_data for docbook docs """ + + user_options = install_data.user_options + + def initialize_options(self): + install_data.initialize_options(self) + self.htmldir = None + + def finalize_options(self): + self.set_undefined_options('install', ('htmldir', 'htmldir')) + install_data.finalize_options(self) + + def run(self): + if not os.path.exists('doc/portage.html'): + self.run_command('docbook') + self.data_files = [ + (self.htmldir, glob.glob('doc/*.html')), + ] + install_data.run(self) + + +class install_epydoc(install_data): + """ install_data for epydoc docs """ + + user_options = install_data.user_options + + def initialize_options(self): + install_data.initialize_options(self) + self.htmldir = None + + def finalize_options(self): + self.set_undefined_options('install', ('htmldir', 'htmldir')) + install_data.finalize_options(self) + + def run(self): + if not os.path.exists('epydoc/index.html'): + self.run_command('epydoc') + self.data_files = [ + (os.path.join(self.htmldir, 'api'), glob.glob('epydoc/*')), + ] + install_data.run(self) + + +class x_build_scripts_custom(build_scripts): + def finalize_options(self): + build_scripts.finalize_options(self) + if 'dir_name' in dir(self): + self.build_dir = os.path.join(self.build_dir, self.dir_name) + if self.dir_name in x_scripts: + self.scripts = x_scripts[self.dir_name] + else: + self.scripts = set(self.scripts) + for other_files in x_scripts.values(): + self.scripts.difference_update(other_files) + + def run(self): + # group scripts by subdirectory + split_scripts = collections.defaultdict(list) + for f in self.scripts: + dir_name = os.path.dirname(f[len('bin/'):]) + split_scripts[dir_name].append(f) + + base_dir = self.build_dir + base_scripts = self.scripts + for d, files in split_scripts.items(): + self.build_dir = os.path.join(base_dir, d) + self.scripts = files + self.copy_scripts() + + # restore previous values + self.build_dir = base_dir + self.scripts = base_scripts + + +class x_build_scripts_bin(x_build_scripts_custom): + dir_name = 'bin' + + +class x_build_scripts_portagebin(x_build_scripts_custom): + dir_name = 'portage' + + +class x_build_scripts(build_scripts): + def initialize_option(self): + build_scripts.initialize_options(self) + + def finalize_options(self): + build_scripts.finalize_options(self) + + def run(self): + self.run_command('build_scripts_bin') + self.run_command('build_scripts_portagebin') + + +class x_clean(clean): + """ clean extended for doc & post-test cleaning """ + + def clean_docs(self): + def get_doc_outfiles(): + for dirpath, dirnames, filenames in os.walk('doc'): + for f in filenames: + if f.endswith('.docbook') or f == 'custom.xsl': + pass + else: + yield os.path.join(dirpath, f) + + # do not recurse + break + + + for f in get_doc_outfiles(): + print('removing %s' % repr(f)) + os.remove(f) + + if os.path.isdir('doc/fragment'): + remove_tree('doc/fragment') + + if os.path.isdir('epydoc'): + remove_tree('epydoc') + + def clean_tests(self): + # do not remove incorrect dirs accidentally + top_dir = os.path.normpath(os.path.join(self.build_lib, '..')) + cprefix = os.path.commonprefix((self.build_base, top_dir)) + if cprefix != self.build_base: + return + + bin_dir = os.path.join(top_dir, 'bin') + if os.path.exists(bin_dir): + remove_tree(bin_dir) + + conf_dir = os.path.join(top_dir, 'cnf') + if os.path.islink(conf_dir): + print('removing %s symlink' % repr(conf_dir)) + os.unlink(conf_dir) + + pni_file = os.path.join(top_dir, '.portage_not_installed') + if os.path.exists(pni_file): + print('removing %s' % repr(pni_file)) + os.unlink(pni_file) + + def clean_man(self): + man_dir = os.path.join(self.build_base, 'man') + if os.path.exists(man_dir): + remove_tree(man_dir) + + def run(self): + if self.all: + self.clean_tests() + self.clean_docs() + self.clean_man() + + clean.run(self) + + +class x_install(install): + """ install command with extra Portage paths """ + + user_options = install.user_options + [ + # note: $prefix and $exec_prefix are reserved for Python install + ('system-prefix=', None, "Prefix for architecture-independent data"), + ('system-exec-prefix=', None, "Prefix for architecture-specific data"), + + ('bindir=', None, "Install directory for main executables"), + ('datarootdir=', None, "Data install root directory"), + ('docdir=', None, "Documentation install directory"), + ('htmldir=', None, "HTML documentation install directory"), + ('mandir=', None, "Manpage root install directory"), + ('portage-base=', 'b', "Portage install base"), + ('portage-bindir=', None, "Install directory for Portage internal-use executables"), + ('portage-datadir=', None, 'Install directory for data files'), + ('sbindir=', None, "Install directory for superuser-intended executables"), + ('sysconfdir=', None, 'System configuration path'), + ] + + # note: the order is important for proper substitution + paths = [ + ('system_prefix', '/usr'), + ('system_exec_prefix', '$system_prefix'), + + ('bindir', '$system_exec_prefix/bin'), + ('sbindir', '$system_exec_prefix/sbin'), + ('sysconfdir', '/etc'), + + ('datarootdir', '$system_prefix/share'), + ('docdir', '$datarootdir/doc/$package-$version'), + ('htmldir', '$docdir/html'), + ('mandir', '$datarootdir/man'), + + ('portage_base', '$system_exec_prefix/lib/portage'), + ('portage_bindir', '$portage_base/bin'), + ('portage_datadir', '$datarootdir/portage'), + + # not customized at the moment + ('logrotatedir', '$sysconfdir/logrotate'), + ('portage_confdir', '$portage_datadir/config'), + ('portage_setsdir', '$portage_confdir/sets'), + ] + + def initialize_options(self): + install.initialize_options(self) + + for key, default in self.paths: + setattr(self, key, default) + self.subst_paths = {} + + def finalize_options(self): + install.finalize_options(self) + + # substitute variables + new_paths = { + 'package': self.distribution.get_name(), + 'version': self.distribution.get_version(), + } + for key, default in self.paths: + new_paths[key] = subst_vars(getattr(self, key), new_paths) + setattr(self, key, new_paths[key]) + self.subst_paths = new_paths + + +class x_install_data(install_data): + """ install_data with customized path support """ + + user_options = install_data.user_options + + def initialize_options(self): + install_data.initialize_options(self) + self.build_base = None + self.paths = None + + def finalize_options(self): + install_data.finalize_options(self) + self.set_undefined_options('build', + ('build_base', 'build_base')) + self.set_undefined_options('install', + ('subst_paths', 'paths')) + + def run(self): + self.run_command('build_man') + + def process_data_files(df): + for d, files in df: + # substitute man sources + if d.startswith('$mandir/'): + files = [os.path.join(self.build_base, v) for v in files] + + # substitute variables in path + d = subst_vars(d, self.paths) + yield (d, files) + + old_data_files = self.data_files + self.data_files = process_data_files(self.data_files) + + install_data.run(self) + self.data_files = old_data_files + + +class x_install_lib(install_lib): + """ install_lib command with Portage path substitution """ + + user_options = install_lib.user_options + + def initialize_options(self): + install_lib.initialize_options(self) + self.portage_base = None + self.portage_bindir = None + self.portage_confdir = None + + def finalize_options(self): + install_lib.finalize_options(self) + self.set_undefined_options('install', + ('portage_base', 'portage_base'), + ('portage_bindir', 'portage_bindir'), + ('portage_confdir', 'portage_confdir')) + + def install(self): + ret = install_lib.install(self) + + def rewrite_file(path, val_dict): + path = os.path.join(self.install_dir, path) + print('Rewriting %s' % path) + with codecs.open(path, 'r', 'utf-8') as f: + data = f.read() + + for varname, val in val_dict.items(): + regexp = r'(?m)^(%s\s*=).*$' % varname + repl = r'\1 %s' % repr(val) + + data = re.sub(regexp, repl, data) + + with codecs.open(path, 'w', 'utf-8') as f: + f.write(data) + + rewrite_file('portage/__init__.py', { + 'VERSION': self.distribution.get_version(), + }) + rewrite_file('portage/const.py', { + 'PORTAGE_BASE_PATH': self.portage_base, + 'PORTAGE_BIN_PATH': self.portage_bindir, + 'PORTAGE_CONFIG_PATH': self.portage_confdir, + }) + + return ret + + +class x_install_scripts_custom(install_scripts): + def initialize_options(self): + install_scripts.initialize_options(self) + self.root = None + + def finalize_options(self): + self.set_undefined_options('install', + ('root', 'root'), + (self.var_name, 'install_dir')) + install_scripts.finalize_options(self) + self.build_dir = os.path.join(self.build_dir, self.dir_name) + + # prepend root + if self.root is not None: + self.install_dir = change_root(self.root, self.install_dir) + + +class x_install_scripts_bin(x_install_scripts_custom): + dir_name = 'bin' + var_name = 'bindir' + + +class x_install_scripts_portagebin(x_install_scripts_custom): + dir_name = 'portage' + var_name = 'portage_bindir' + + +class x_install_scripts(install_scripts): + def initialize_option(self): + pass + + def finalize_options(self): + pass + + def run(self): + self.run_command('install_scripts_bin') + self.run_command('install_scripts_portagebin') + + +class x_sdist(sdist): + """ sdist defaulting to .tar.bz2 format """ + + def finalize_options(self): + if self.formats is None: + self.formats = ['bztar'] + + sdist.finalize_options(self) + + +class build_tests(x_build_scripts_custom): + """ Prepare build dir for running tests. """ + + def initialize_options(self): + x_build_scripts_custom.initialize_options(self) + self.build_base = None + self.build_lib = None + + def finalize_options(self): + x_build_scripts_custom.finalize_options(self) + self.set_undefined_options('build', + ('build_base', 'build_base'), + ('build_lib', 'build_lib')) + + # since we will be writing to $build_lib/.., it is important + # that we do not leave $build_base + self.top_dir = os.path.normpath(os.path.join(self.build_lib, '..')) + cprefix = os.path.commonprefix((self.build_base, self.top_dir)) + if cprefix != self.build_base: + raise SystemError('build_lib must be a subdirectory of build_base') + + self.build_dir = os.path.join(self.top_dir, 'bin') + + def run(self): + self.run_command('build_py') + + # install all scripts $build_lib/../bin + # (we can't do a symlink since we want shebangs corrected) + x_build_scripts_custom.run(self) + + # symlink 'cnf' directory + conf_dir = os.path.join(self.top_dir, 'cnf') + if os.path.exists(conf_dir): + if not os.path.islink(conf_dir): + raise SystemError('%s exists and is not a symlink (collision)' + % repr(conf_dir)) + os.unlink(conf_dir) + conf_src = os.path.relpath('cnf', self.top_dir) + print('Symlinking %s -> %s' % (conf_dir, conf_src)) + os.symlink(conf_src, conf_dir) + + # create $build_lib/../.portage_not_installed + # to enable proper paths in tests + with open(os.path.join(self.top_dir, '.portage_not_installed'), 'w') as f: + pass + + +class test(Command): + """ run tests """ + + user_options = [] + + def initialize_options(self): + self.build_lib = None + + def finalize_options(self): + self.set_undefined_options('build', + ('build_lib', 'build_lib')) + + def run(self): + self.run_command('build_tests') + subprocess.check_call([ + sys.executable, '-bWd', + os.path.join(self.build_lib, 'portage/tests/runTests.py') + ]) + + +def find_packages(): + for dirpath, dirnames, filenames in os.walk('pym'): + if '__init__.py' in filenames: + yield os.path.relpath(dirpath, 'pym') + + +def find_scripts(): + for dirpath, dirnames, filenames in os.walk('bin'): + for f in filenames: + if f not in ['deprecated-path']: + yield os.path.join(dirpath, f) + + +def get_manpages(): + linguas = os.environ.get('LINGUAS') + if linguas is not None: + linguas = linguas.split() + + for dirpath, dirnames, filenames in os.walk('man'): + groups = collections.defaultdict(list) + for f in filenames: + fn, suffix = f.rsplit('.', 1) + groups[suffix].append(os.path.join(dirpath, f)) + + topdir = dirpath[len('man/'):] + if not topdir or linguas is None or topdir in linguas: + for g, mans in groups.items(): + yield [os.path.join('$mandir', topdir, 'man%s' % g), mans] + +setup( + name = 'portage', + version = '2.2.14_rc1', + url = 'https://wiki.gentoo.org/wiki/Project:Portage', + author = 'Gentoo Portage Development Team', + author_email = 'dev-portage@gentoo.org', + + package_dir = {'': 'pym'}, + packages = list(find_packages()), + # something to cheat build & install commands + scripts = list(find_scripts()), + + data_files = list(get_manpages()) + [ + ['$sysconfdir', ['cnf/etc-update.conf', 'cnf/dispatch-conf.conf']], + ['$logrotatedir', ['cnf/logrotate.d/elog-save-summary']], + ['$portage_confdir', [ + 'cnf/make.conf.example', 'cnf/make.globals', 'cnf/repos.conf']], + ['$portage_setsdir', ['cnf/sets/portage.conf']], + ['$docdir', ['NEWS', 'RELEASE-NOTES']], + ['$portage_base/bin', ['bin/deprecated-path']], + ], + + cmdclass = { + 'build': x_build, + 'build_man': build_man, + 'build_scripts': x_build_scripts, + 'build_scripts_bin': x_build_scripts_bin, + 'build_scripts_portagebin': x_build_scripts_portagebin, + 'build_tests': build_tests, + 'clean': x_clean, + 'docbook': docbook, + 'epydoc': epydoc, + 'install': x_install, + 'install_data': x_install_data, + 'install_docbook': install_docbook, + 'install_epydoc': install_epydoc, + 'install_lib': x_install_lib, + 'install_scripts': x_install_scripts, + 'install_scripts_bin': x_install_scripts_bin, + 'install_scripts_portagebin': x_install_scripts_portagebin, + 'sdist': x_sdist, + 'test': test, + }, + + classifiers = [ + 'Development Status :: 5 - Production/Stable', + 'Environment :: Console', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)', + 'Operating System :: POSIX', + 'Programming Language :: Python', + 'Topic :: System :: Installation/Setup' + ] +) diff --git a/testpath b/testpath new file mode 100644 index 000000000..144387dca --- /dev/null +++ b/testpath @@ -0,0 +1,11 @@ +# +# Generic testpath to source. This sets PATH and PYTHONPATH +# to use the current checkout code, before searching the main PATH, PYTHONPATH +# cd ~/.../portage +# source ./testpath +# emerge foo +# repoman manifest +# ... +export PATH="$(dirname ${BASH_SOURCE[0]})/bin:${PATH}" +export PYTHONPATH=PYTHONPATH="$(dirname $BASH_SOURCE[0])/pym:${PYTHONPATH:+:}${PYTHONPATH}" +export PYTHONWARNINGS=d,i::ImportWarning |