aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorZac Medico <zmedico@gentoo.org>2012-08-21 20:22:19 (GMT)
committerZac Medico <zmedico@gentoo.org>2012-08-21 20:22:19 (GMT)
commitaccee1b7c61da284022f86d9ab39bcb492ea4023 (patch)
tree7a25fa56dbb0b3878f5a8050f60f0a193281057b
parentPollScheduler: use timeout for loadavg checks (diff)
downloadportage-accee1b7c61da284022f86d9ab39bcb492ea4023.zip
portage-accee1b7c61da284022f86d9ab39bcb492ea4023.tar.gz
portage-accee1b7c61da284022f86d9ab39bcb492ea4023.tar.bz2
Implement PORTAGE_CHECKSUM_FILTER for bug #432170
-rw-r--r--man/make.conf.519
-rw-r--r--pym/_emerge/EbuildFetcher.py4
-rw-r--r--pym/portage/checksum.py54
-rw-r--r--pym/portage/dbapi/bintree.py6
-rw-r--r--pym/portage/manifest.py15
-rw-r--r--pym/portage/package/ebuild/_config/special_env_vars.py1
-rw-r--r--pym/portage/package/ebuild/digestcheck.py13
-rw-r--r--pym/portage/package/ebuild/fetch.py13
8 files changed, 107 insertions, 18 deletions
diff --git a/man/make.conf.5 b/man/make.conf.5
index c617fbc..59f32b8 100644
--- a/man/make.conf.5
+++ b/man/make.conf.5
@@ -1,4 +1,4 @@
-.TH "MAKE.CONF" "5" "Jul 2012" "Portage VERSION" "Portage"
+.TH "MAKE.CONF" "5" "Aug 2012" "Portage VERSION" "Portage"
.SH "NAME"
make.conf \- custom settings for Portage
.SH "SYNOPSIS"
@@ -697,6 +697,23 @@ for bzip2 compression operations. \fBPORTAGE_BZIP2_COMMAND\fR will also be
called for extraction operation, with -d appended, unless the
\fBPORTAGE_BUNZIP2_COMMAND\fR variable is set.
.TP
+\fBPORTAGE_CHECKSUM_FILTER\fR = \fI[space delimited list of hash names]\fR
+This variable may be used to filter the hash functions that are used to
+verify integrity of files. Hash functions names are case\-insensitive, and
+the \fI*\fR and \fI\-*\fR wildcard tokens are supported.
+.br
+Defaults to the value of *.
+.br
+.I Examples:
+.nf
+# Use all available hash functions
+PORTAGE_CHECKSUM_FILTER="*"
+# Use any function except whirlpool
+PORTAGE_CHECKSUM_FILTER="* \-whirlpool"
+# Only use sha256
+PORTAGE_CHECKSUM_FILTER="\-* sha256"
+.fi
+.TP
\fBPORTAGE_COMPRESS\fR = \fI"bzip2"\fR
This variable contains the command used to compress documentation during the
install phase.
diff --git a/pym/_emerge/EbuildFetcher.py b/pym/_emerge/EbuildFetcher.py
index c0a7fdd..bbcb6a9 100644
--- a/pym/_emerge/EbuildFetcher.py
+++ b/pym/_emerge/EbuildFetcher.py
@@ -13,6 +13,7 @@ from portage import os
from portage import _encodings
from portage import _unicode_encode
from portage import _unicode_decode
+from portage.checksum import _hash_filter
from portage.elog.messages import eerror
from portage.package.ebuild.fetch import _check_distfile, fetch
from portage.util._pty import _create_pty_or_pipe
@@ -57,6 +58,7 @@ class EbuildFetcher(SpawnProcess):
if st.st_size != expected_size:
return False
+ hash_filter = _hash_filter(settings.get("PORTAGE_CHECKSUM_FILTER", ""))
stdout_orig = sys.stdout
stderr_orig = sys.stderr
global_havecolor = portage.output.havecolor
@@ -78,7 +80,7 @@ class EbuildFetcher(SpawnProcess):
break
continue
ok, st = _check_distfile(os.path.join(distdir, filename),
- mydigests, eout, show_errors=False)
+ mydigests, eout, show_errors=False, hash_filter=hash_filter)
if not ok:
success = False
break
diff --git a/pym/portage/checksum.py b/pym/portage/checksum.py
index daf4a0c..de4cc66 100644
--- a/pym/portage/checksum.py
+++ b/pym/portage/checksum.py
@@ -217,6 +217,60 @@ def _filter_unaccelarated_hashes(digests):
return digests
+class _hash_filter(object):
+ """
+ Implements filtering for PORTAGE_CHECKSUM_FILTER.
+ """
+
+ __slots__ = ('transparent', '_tokens',)
+
+ def __init__(self, filter_str):
+ tokens = filter_str.upper().split()
+ if not tokens or tokens[-1] == "*":
+ del tokens[:]
+ self.transparent = not tokens
+ tokens.reverse()
+ self._tokens = tuple(tokens)
+
+ def __call__(self, hash_name):
+ if self.transparent:
+ return True
+ matches = ("*", hash_name)
+ for token in self._tokens:
+ if token in matches:
+ return True
+ elif token[:1] == "-":
+ if token[1:] in matches:
+ return False
+ return False
+
+def _apply_hash_filter(digests, hash_filter):
+ """
+ Return a new dict containing the filtered digests, or the same
+ dict if no changes are necessary. This will always preserve at
+ at least one digest, in order to ensure that they are not all
+ discarded.
+ """
+ if hash_filter.transparent:
+ return digests
+
+ verifiable_hash_types = set(digests).intersection(hashfunc_map)
+ verifiable_hash_types.discard("size")
+ modified = False
+ if len(verifiable_hash_types) > 1:
+ for k in list(verifiable_hash_types):
+ if not hash_filter(k):
+ modified = True
+ verifiable_hash_types.remove(k)
+ if len(verifiable_hash_types) == 1:
+ break
+
+ if modified:
+ digests = dict((k, v) for (k, v) in digests.items()
+ if k == "size" or k in verifiable_hash_types)
+
+ return digests
+
def verify_all(filename, mydict, calc_prelink=0, strict=0):
"""
Verify all checksums against a file.
diff --git a/pym/portage/dbapi/bintree.py b/pym/portage/dbapi/bintree.py
index 6c01867..a2fd5ea 100644
--- a/pym/portage/dbapi/bintree.py
+++ b/pym/portage/dbapi/bintree.py
@@ -5,7 +5,8 @@ __all__ = ["bindbapi", "binarytree"]
import portage
portage.proxy.lazyimport.lazyimport(globals(),
- 'portage.checksum:hashfunc_map,perform_multiple_checksums,verify_all',
+ 'portage.checksum:hashfunc_map,perform_multiple_checksums,' + \
+ 'verify_all,_apply_hash_filter,_hash_filter',
'portage.dbapi.dep_expand:dep_expand',
'portage.dep:dep_getkey,isjustname,isvalidatom,match_from_list',
'portage.output:EOutput,colorize',
@@ -1462,6 +1463,9 @@ class binarytree(object):
if not digests:
return False
+ hash_filter = _hash_filter(
+ self.settings.get("PORTAGE_CHECKSUM_FILTER", ""))
+ digests = _apply_hash_filter(digests, hash_filter)
eout = EOutput()
eout.quiet = self.settings.get("PORTAGE_QUIET") == "1"
ok, st = _check_distfile(pkg_path, digests, eout, show_errors=0)
diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py
index a04b717..b2f1ff2 100644
--- a/pym/portage/manifest.py
+++ b/pym/portage/manifest.py
@@ -9,7 +9,7 @@ import warnings
import portage
portage.proxy.lazyimport.lazyimport(globals(),
'portage.checksum:hashfunc_map,perform_multiple_checksums,' + \
- 'verify_all,_filter_unaccelarated_hashes',
+ 'verify_all,_apply_hash_filter,_filter_unaccelarated_hashes',
'portage.util:write_atomic',
)
@@ -502,14 +502,17 @@ class Manifest(object):
for t in MANIFEST2_IDENTIFIERS:
self.checkTypeHashes(t, ignoreMissingFiles=ignoreMissingFiles)
- def checkTypeHashes(self, idtype, ignoreMissingFiles=False):
+ def checkTypeHashes(self, idtype, ignoreMissingFiles=False, hash_filter=None):
for f in self.fhashdict[idtype]:
- self.checkFileHashes(idtype, f, ignoreMissing=ignoreMissingFiles)
+ self.checkFileHashes(idtype, f, ignoreMissing=ignoreMissingFiles,
+ hash_filter=hash_filter)
- def checkFileHashes(self, ftype, fname, ignoreMissing=False):
+ def checkFileHashes(self, ftype, fname, ignoreMissing=False, hash_filter=None):
+ digests = _filter_unaccelarated_hashes(self.fhashdict[ftype][fname])
+ if hash_filter is not None:
+ digests = _apply_hash_filter(digests, hash_filter)
try:
- ok, reason = verify_all(self._getAbsname(ftype, fname),
- _filter_unaccelarated_hashes(self.fhashdict[ftype][fname]))
+ ok, reason = verify_all(self._getAbsname(ftype, fname), digests)
if not ok:
raise DigestException(tuple([self._getAbsname(ftype, fname)]+list(reason)))
return ok, reason
diff --git a/pym/portage/package/ebuild/_config/special_env_vars.py b/pym/portage/package/ebuild/_config/special_env_vars.py
index 6ed6d05..c974eb9 100644
--- a/pym/portage/package/ebuild/_config/special_env_vars.py
+++ b/pym/portage/package/ebuild/_config/special_env_vars.py
@@ -150,6 +150,7 @@ environ_filter += [
"PORTAGE_BACKGROUND", "PORTAGE_BACKGROUND_UNMERGE",
"PORTAGE_BINHOST",
"PORTAGE_BINHOST_CHUNKSIZE", "PORTAGE_BUILDIR_LOCKED",
+ "PORTAGE_CHECKSUM_FILTER",
"PORTAGE_ELOG_CLASSES",
"PORTAGE_ELOG_MAILFROM", "PORTAGE_ELOG_MAILSUBJECT",
"PORTAGE_ELOG_MAILURI", "PORTAGE_ELOG_SYSTEM",
diff --git a/pym/portage/package/ebuild/digestcheck.py b/pym/portage/package/ebuild/digestcheck.py
index 8705639..1d59948 100644
--- a/pym/portage/package/ebuild/digestcheck.py
+++ b/pym/portage/package/ebuild/digestcheck.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2011 Gentoo Foundation
+# Copyright 2010-2012 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
__all__ = ['digestcheck']
@@ -6,6 +6,7 @@ __all__ = ['digestcheck']
import warnings
from portage import os, _encodings, _unicode_decode
+from portage.checksum import _hash_filter
from portage.exception import DigestException, FileNotFound
from portage.localization import _
from portage.output import EOutput
@@ -28,6 +29,7 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None):
if mysettings.get("EBUILD_SKIP_MANIFEST") == "1":
return 1
pkgdir = mysettings["O"]
+ hash_filter = _hash_filter(mysettings.get("PORTAGE_CHECKSUM_FILTER", ""))
if mf is None:
mf = mysettings.repositories.get_repo_for_location(
os.path.dirname(os.path.dirname(pkgdir)))
@@ -38,15 +40,16 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None):
if not mf.thin and strict and "PORTAGE_PARALLEL_FETCHONLY" not in mysettings:
if mf.fhashdict.get("EBUILD"):
eout.ebegin(_("checking ebuild checksums ;-)"))
- mf.checkTypeHashes("EBUILD")
+ mf.checkTypeHashes("EBUILD", hash_filter=hash_filter)
eout.eend(0)
if mf.fhashdict.get("AUX"):
eout.ebegin(_("checking auxfile checksums ;-)"))
- mf.checkTypeHashes("AUX")
+ mf.checkTypeHashes("AUX", hash_filter=hash_filter)
eout.eend(0)
if mf.fhashdict.get("MISC"):
eout.ebegin(_("checking miscfile checksums ;-)"))
- mf.checkTypeHashes("MISC", ignoreMissingFiles=True)
+ mf.checkTypeHashes("MISC", ignoreMissingFiles=True,
+ hash_filter=hash_filter)
eout.eend(0)
for f in myfiles:
eout.ebegin(_("checking %s ;-)") % f)
@@ -58,7 +61,7 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None):
writemsg(_("\n!!! Missing digest for '%s'\n") % (f,),
noiselevel=-1)
return 0
- mf.checkFileHashes(ftype, f)
+ mf.checkFileHashes(ftype, f, hash_filter=hash_filter)
eout.eend(0)
except FileNotFound as e:
eout.eend(1)
diff --git a/pym/portage/package/ebuild/fetch.py b/pym/portage/package/ebuild/fetch.py
index 60ed04d..8365ad2 100644
--- a/pym/portage/package/ebuild/fetch.py
+++ b/pym/portage/package/ebuild/fetch.py
@@ -26,7 +26,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
from portage import OrderedDict, os, selinux, shutil, _encodings, \
_shell_quote, _unicode_encode
from portage.checksum import (hashfunc_map, perform_md5, verify_all,
- _filter_unaccelarated_hashes)
+ _filter_unaccelarated_hashes, _hash_filter, _apply_hash_filter)
from portage.const import BASH_BINARY, CUSTOM_MIRRORS_FILE, \
GLOBAL_CONFIG_PATH
from portage.data import portage_gid, portage_uid, secpass, userpriv_groups
@@ -185,7 +185,7 @@ def _check_digests(filename, digests, show_errors=1):
return False
return True
-def _check_distfile(filename, digests, eout, show_errors=1):
+def _check_distfile(filename, digests, eout, show_errors=1, hash_filter=None):
"""
@return a tuple of (match, stat_obj) where match is True if filename
matches all given digests (if any) and stat_obj is a stat result, or
@@ -212,6 +212,8 @@ def _check_distfile(filename, digests, eout, show_errors=1):
return (False, st)
else:
digests = _filter_unaccelarated_hashes(digests)
+ if hash_filter is not None:
+ digests = _apply_hash_filter(digests, hash_filter)
if _check_digests(filename, digests, show_errors=show_errors):
eout.ebegin("%s %s ;-)" % (os.path.basename(filename),
" ".join(sorted(digests))))
@@ -355,6 +357,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
if try_mirrors:
mymirrors += [x.rstrip("/") for x in mysettings["GENTOO_MIRRORS"].split() if x]
+ hash_filter = _hash_filter(mysettings.get("PORTAGE_CHECKSUM_FILTER", ""))
skip_manifest = mysettings.get("EBUILD_SKIP_MANIFEST") == "1"
if skip_manifest:
allow_missing_digests = True
@@ -637,7 +640,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
eout = EOutput()
eout.quiet = mysettings.get("PORTAGE_QUIET") == "1"
match, mystat = _check_distfile(
- myfile_path, pruned_digests, eout)
+ myfile_path, pruned_digests, eout, hash_filter=hash_filter)
if match:
# Skip permission adjustment for symlinks, since we don't
# want to modify anything outside of the primary DISTDIR,
@@ -709,7 +712,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
for x in ro_distdirs:
filename = os.path.join(x, myfile)
match, mystat = _check_distfile(
- filename, pruned_digests, eout)
+ filename, pruned_digests, eout, hash_filter=hash_filter)
if match:
readonly_file = filename
break
@@ -796,6 +799,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
continue
else:
digests = _filter_unaccelarated_hashes(mydigests[myfile])
+ digests = _apply_hash_filter(digests, hash_filter)
verified_ok, reason = verify_all(myfile_path, digests)
if not verified_ok:
writemsg(_("!!! Previously fetched"
@@ -1053,6 +1057,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
# net connection. This way we have a chance to try to download
# from another mirror...
digests = _filter_unaccelarated_hashes(mydigests[myfile])
+ digests = _apply_hash_filter(digests, hash_filter)
verified_ok, reason = verify_all(myfile_path, digests)
if not verified_ok:
writemsg(_("!!! Fetched file: %s VERIFY FAILED!\n") % myfile,