cache: rewrite to support arbitrary validation method

Specifically, the cache can use any portage supported checksum method,
or use the standard mtime approach.  In addition, support controlling
whether or not paths are stored, and generally try to restore some
of the centralization/encapsulation that was in place originally.
(cherry picked from commit bc1aed614fb588f0ade5bcb5d1265a8db0f8d247)

Change-Id: Ic38057e7dbb15063c64a93c99e66e113a7d4c70e

2 files changed, 53 insertions, 17 deletions

diff --git a/pym/portage/cache/metadata.py b/pym/portage/cache/metadata.py
index 4c735d7e3..07ec20ebc 100644
--- a/pym/portage/cache/metadata.py
+++ b/pym/portage/cache/metadata.py
@@ -6,6 +6,7 @@ import errno
 import re
 import stat
 import sys
+from operator import attrgetter
 from portage import os
 from portage import _encodings
 from portage import _unicode_encode
@@ -63,9 +64,11 @@ class database(flat_hash.database):
 			if "INHERITED" in d:
 				if self.ec is None:
 					self.ec = portage.eclass_cache.cache(self.location[:-15])
+				getter = attrgetter(self.validation_chf)
 				try:
-					d["_eclasses_"] = self.ec.get_eclass_data(
-						d["INHERITED"].split())
+					ec_data = self.ec.get_eclass_data(d["INHERITED"].split())
+					d["_eclasses_"] = dict((k, (v.eclass_dir, getter(v)))
+						for k,v in ec_data.items())
 				except KeyError as e:
 					# INHERITED contains a non-existent eclass.
 					raise cache_errors.CacheCorruption(cpv, e)
diff --git a/pym/portage/cache/template.py b/pym/portage/cache/template.py
index f84d8f4b9..a76a5f59f 100644
--- a/pym/portage/cache/template.py
+++ b/pym/portage/cache/template.py
@@ -7,6 +7,7 @@ from portage.cache.cache_errors import InvalidRestriction
 from portage.cache.mappings import ProtectedDict
 import sys
 import warnings
+import operator
 
 if sys.hexversion >= 0x3000000:
 	basestring = str
@@ -21,6 +22,8 @@ class database(object):
 	autocommits = False
 	cleanse_keys = False
 	serialize_eclasses = True
+	validation_chf = 'mtime'
+	store_eclass_paths = True
 
 	def __init__(self, location, label, auxdbkeys, readonly=False):
 		""" initialize the derived class; specifically, store label/keys"""
@@ -40,7 +43,8 @@ class database(object):
 			self.updates = 0
 		d=self._getitem(cpv)
 		if self.serialize_eclasses and "_eclasses_" in d:
-			d["_eclasses_"] = reconstruct_eclasses(cpv, d["_eclasses_"])
+			d["_eclasses_"] = reconstruct_eclasses(cpv, d["_eclasses_"],
+				self.validation_chf, paths=self.store_eclass_paths)
 		elif "_eclasses_" not in d:
 			d["_eclasses_"] = {}
 		mtime = d.get('_mtime_')
@@ -71,10 +75,12 @@ class database(object):
 				if not v:
 					del d[k]
 			if self.serialize_eclasses and "_eclasses_" in values:
-				d["_eclasses_"] = serialize_eclasses(d["_eclasses_"])
+				d["_eclasses_"] = serialize_eclasses(d["_eclasses_"],
+					self.validation_chf, paths=self.store_eclass_paths)
 		elif self.serialize_eclasses and "_eclasses_" in values:
 			d = ProtectedDict(values)
-			d["_eclasses_"] = serialize_eclasses(d["_eclasses_"])
+			d["_eclasses_"] = serialize_eclasses(d["_eclasses_"],
+				self.validation_chf, paths=self.store_eclass_paths)
 		else:
 			d = values
 		self._setitem(cpv, d)
@@ -159,6 +165,18 @@ class database(object):
 		except KeyError:
 			return x
 
+	def validate_entry(self, entry, ebuild_hash, eclass_db):
+		hash_key = '_%s_' % self.validation_chf
+		if entry[hash_key] != getattr(ebuild_hash, self.validation_chf):
+			return False
+		update = eclass_db.validate_and_rewrite_cache(entry['_eclasses_'], self.validation_chf,
+			self.store_eclass_paths)
+		if update is None:
+			return False
+		if update:
+			entry['_eclasses_'] = update
+		return True
+
 	def get_matches(self, match_dict):
 		"""generic function for walking the entire cache db, matching restrictions to
 		filter what cpv's are returned.  Derived classes should override this if they
@@ -195,7 +213,9 @@ class database(object):
 		keys = __iter__
 		items = iteritems
 
-def serialize_eclasses(eclass_dict):
+_keysorter = operator.itemgetter(0)
+
+def serialize_eclasses(eclass_dict, chf_type='mtime', paths=True):
 	"""takes a dict, returns a string representing said dict"""
 	"""The "new format", which causes older versions of <portage-2.1.2 to
 	traceback with a ValueError due to failed long() conversion.  This format
@@ -206,27 +226,40 @@ def serialize_eclasses(eclass_dict):
 	"""
 	if not eclass_dict:
 		return ""
-	return "\t".join(k + "\t%s\t%s" % eclass_dict[k] \
-		for k in sorted(eclass_dict))
+	getter = operator.attrgetter(chf_type)
+	if paths:
+		return "\t".join("%s\t%s\t%s" % (k, v.eclass_dir, getter(v))
+			for k, v in sorted(eclass_dict.items(), key=_keysorter))
+	return "\t".join("%s\t%s" % (k, getter(v))
+		for k, v in sorted(eclass_dict.items(), key=_keysorter))
 
-def reconstruct_eclasses(cpv, eclass_string):
+
+def reconstruct_eclasses(cpv, eclass_string, chf_type='mtime', paths=True):
 	"""returns a dict when handed a string generated by serialize_eclasses"""
 	eclasses = eclass_string.rstrip().lstrip().split("\t")
 	if eclasses == [""]:
 		# occasionally this occurs in the fs backends.  they suck.
 		return {}
-	
-	if len(eclasses) % 2 != 0 and len(eclasses) % 3 != 0:
+
+	converter = str
+	if chf_type == 'mtime':
+		converter = long
+
+	if paths:
+		if len(eclasses) % 3 != 0:
+			raise cache_errors.CacheCorruption(cpv, "_eclasses_ was of invalid len %i" % len(eclasses))
+	elif len(eclasses) % 2 != 0:
 		raise cache_errors.CacheCorruption(cpv, "_eclasses_ was of invalid len %i" % len(eclasses))
 	d={}
 	try:
-		if eclasses[1].isdigit():
-			for x in range(0, len(eclasses), 2):
-				d[eclasses[x]] = ("", long(eclasses[x + 1]))
-		else:
+		i = iter(eclasses)
+		if paths:
 			# The old format contains paths that will be discarded.
-			for x in range(0, len(eclasses), 3):
-				d[eclasses[x]] = (eclasses[x + 1], long(eclasses[x + 2]))
+			for name, path, val in zip(i, i, i):
+				d[name] = (path, converter(val))
+		else:
+			for name, val in zip(i, i):
+				d[name] = converter(val)
 	except IndexError:
 		raise cache_errors.CacheCorruption(cpv,
 			"_eclasses_ was of invalid len %i" % len(eclasses))