emirrordist: Implement mirror layout.conf support

Read mirror layout from layout.conf in distfiles directory.  When
multiple layouts are used, fetch according to the first one and then
hardlink according to the remaining layouts.  Similarly, when deleting
recycle or delete according to the first layout, then unlink according
to all remaining layouts.

Reviewed-by: Zac Medico <zmedico@gentoo.org>
Closes: https://github.com/gentoo/portage/pull/463
Signed-off-by: Michał Górny <mgorny@gentoo.org>

6 files changed, 139 insertions, 34 deletions

diff --git a/lib/portage/_emirrordist/Config.py b/lib/portage/_emirrordist/Config.py
index 574a83559..ace3b0a6a 100644
--- a/lib/portage/_emirrordist/Config.py
+++ b/lib/portage/_emirrordist/Config.py
@@ -1,4 +1,4 @@
-# Copyright 2013 Gentoo Foundation
+# Copyright 2013-2019 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import copy
@@ -10,6 +10,7 @@ import time
 
 import portage
 from portage import os
+from portage.package.ebuild.fetch import MirrorLayoutConfig
 from portage.util import grabdict, grablines
 from portage.util._ShelveUnicodeWrapper import ShelveUnicodeWrapper
 
@@ -72,6 +73,11 @@ class Config(object):
 			self.deletion_db = self._open_shelve(
 				options.deletion_db, 'deletion')
 
+		self.layout_conf = MirrorLayoutConfig()
+		self.layout_conf.read_from_file(
+				os.path.join(self.distfiles, 'layout.conf'))
+		self.layouts = self.layout_conf.get_all_layouts()
+
 	def _open_log(self, log_desc, log_path, mode):
 
 		if log_path is None or self.options.dry_run:
diff --git a/lib/portage/_emirrordist/DeletionIterator.py b/lib/portage/_emirrordist/DeletionIterator.py
index dff52c042..cd773b3c8 100644
--- a/lib/portage/_emirrordist/DeletionIterator.py
+++ b/lib/portage/_emirrordist/DeletionIterator.py
@@ -1,4 +1,4 @@
-# Copyright 2013 Gentoo Foundation
+# Copyright 2013-2019 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import logging
@@ -20,17 +20,28 @@ class DeletionIterator(object):
 		deletion_db = self._config.deletion_db
 		deletion_delay = self._config.options.deletion_delay
 		start_time = self._config.start_time
-		distfiles_set = set(os.listdir(self._config.options.distfiles))
+		distfiles_set = set()
+		for layout in self._config.layouts:
+			distfiles_set.update(layout.get_filenames(distdir))
 		for filename in distfiles_set:
-			try:
-				st = os.stat(os.path.join(distdir, filename))
-			except OSError as e:
-				logging.error("stat failed on '%s' in distfiles: %s\n" %
-					(filename, e))
-				continue
-			if not stat.S_ISREG(st.st_mode):
+			# require at least one successful stat()
+			first_exception = None
+			for layout in reversed(self._config.layouts):
+				try:
+					st = os.stat(
+							os.path.join(distdir, layout.get_path(filename)))
+				except OSError as e:
+					first_exception = e
+				else:
+					if stat.S_ISREG(st.st_mode):
+						break
+			else:
+				if first_exception is not None:
+					logging.error("stat failed on '%s' in distfiles: %s\n" %
+						(filename, first_exception))
 				continue
-			elif filename in file_owners:
+
+			if filename in file_owners:
 				if deletion_db is not None:
 					try:
 						del deletion_db[filename]
diff --git a/lib/portage/_emirrordist/DeletionTask.py b/lib/portage/_emirrordist/DeletionTask.py
index 7d10957fa..db5ac5ffb 100644
--- a/lib/portage/_emirrordist/DeletionTask.py
+++ b/lib/portage/_emirrordist/DeletionTask.py
@@ -1,4 +1,4 @@
-# Copyright 2013 Gentoo Foundation
+# Copyright 2013-2019 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import errno
@@ -15,10 +15,10 @@ class DeletionTask(CompositeTask):
 	def _start(self):
 
 		distfile_path = os.path.join(
-			self.config.options.distfiles, self.distfile)
+			self.config.options.distfiles,
+			self.config.layouts[0].get_path(self.distfile))
 
 		if self.config.options.recycle_dir is not None:
-			distfile_path = os.path.join(self.config.options.distfiles, self.distfile)
 			recycle_path = os.path.join(
 				self.config.options.recycle_dir, self.distfile)
 			if self.config.options.dry_run:
@@ -28,13 +28,14 @@ class DeletionTask(CompositeTask):
 				logging.debug(("move '%s' from "
 					"distfiles to recycle") % self.distfile)
 				try:
-					os.rename(distfile_path, recycle_path)
+					# note: distfile_path can be a symlink here
+					os.rename(os.path.realpath(distfile_path), recycle_path)
 				except OSError as e:
 					if e.errno != errno.EXDEV:
 						logging.error(("rename %s from distfiles to "
 							"recycle failed: %s") % (self.distfile, e))
 				else:
-					self.returncode = os.EX_OK
+					self._delete_links()
 					self._async_wait()
 					return
 
@@ -62,8 +63,7 @@ class DeletionTask(CompositeTask):
 					success = False
 
 		if success:
-			self._success()
-			self.returncode = os.EX_OK
+			self._delete_links()
 		else:
 			self.returncode = 1
 
@@ -93,14 +93,33 @@ class DeletionTask(CompositeTask):
 			success = False
 
 		if success:
-			self._success()
-			self.returncode = os.EX_OK
+			self._delete_links()
 		else:
 			self.returncode = 1
 
 		self._current_task = None
 		self.wait()
 
+	def _delete_links(self):
+		success = True
+		for layout in self.config.layouts[1:]:
+			distfile_path = os.path.join(
+				self.config.options.distfiles,
+				layout.get_path(self.distfile))
+			try:
+				os.unlink(distfile_path)
+			except OSError as e:
+				if e.errno not in (errno.ENOENT, errno.ESTALE):
+					logging.error("%s unlink failed in distfiles: %s" %
+						(self.distfile, e))
+					success = False
+
+		if success:
+			self._success()
+			self.returncode = os.EX_OK
+		else:
+			self.returncode = 1
+
 	def _success(self):
 
 		cpv = "unknown"
diff --git a/lib/portage/_emirrordist/FetchTask.py b/lib/portage/_emirrordist/FetchTask.py
index 1440b697c..7b68b7d3e 100644
--- a/lib/portage/_emirrordist/FetchTask.py
+++ b/lib/portage/_emirrordist/FetchTask.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2014 Gentoo Foundation
+# Copyright 2013-2019 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import division
@@ -14,6 +14,7 @@ import sys
 import portage
 from portage import _encodings, _unicode_encode
 from portage import os
+from portage.util import ensure_dirs
 from portage.util._async.FileCopier import FileCopier
 from portage.util._async.FileDigester import FileDigester
 from portage.util._async.PipeLogger import PipeLogger
@@ -61,7 +62,8 @@ class FetchTask(CompositeTask):
 			return
 
 		distfile_path = os.path.join(
-			self.config.options.distfiles, self.distfile)
+			self.config.options.distfiles,
+			self.config.layouts[0].get_path(self.distfile))
 
 		st = None
 		size_ok = False
@@ -332,7 +334,8 @@ class FetchTask(CompositeTask):
 				return
 			else:
 				src = os.path.join(current_mirror.location, self.distfile)
-				dest = os.path.join(self.config.options.distfiles, self.distfile)
+				dest = os.path.join(self.config.options.distfiles,
+						self.config.layouts[0].get_path(self.distfile))
 				if self._hardlink_atomic(src, dest,
 					"%s to %s" % (current_mirror.name, "distfiles")):
 					logging.debug("hardlink '%s' from %s to distfiles" %
@@ -501,7 +504,9 @@ class FetchTask(CompositeTask):
 				except OSError:
 					pass
 			else:
-				dest = os.path.join(self.config.options.distfiles, self.distfile)
+				dest = os.path.join(self.config.options.distfiles,
+						self.config.layouts[0].get_path(self.distfile))
+				ensure_dirs(os.path.dirname(dest))
 				try:
 					os.rename(self._fetch_tmp_file, dest)
 				except OSError:
@@ -514,9 +519,7 @@ class FetchTask(CompositeTask):
 						self._fetch_copier_exit)
 					return
 				else:
-					self._success()
-					self.returncode = os.EX_OK
-					self.wait()
+					self._make_layout_links()
 					return
 
 		self._try_next_mirror()
@@ -535,9 +538,7 @@ class FetchTask(CompositeTask):
 			return
 
 		if copier.returncode == os.EX_OK:
-			self._success()
-			self.returncode = os.EX_OK
-			self.wait()
+			self._make_layout_links()
 		else:
 			# out of space?
 			msg = "%s %s copy failed unexpectedly" % \
@@ -551,6 +552,38 @@ class FetchTask(CompositeTask):
 			self.returncode = 1
 			self.wait()
 
+	def _make_layout_links(self):
+		dist_path = None
+		success = True
+		for layout in self.config.layouts[1:]:
+			if dist_path is None:
+				dist_path = os.path.join(self.config.options.distfiles,
+						self.config.layouts[0].get_path(self.distfile))
+			link_path = os.path.join(self.config.options.distfiles,
+					layout.get_path(self.distfile))
+			ensure_dirs(os.path.dirname(link_path))
+			src_path = dist_path
+			if self.config.options.symlinks:
+				src_path = os.path.relpath(dist_path,
+						os.path.dirname(link_path))
+
+			if not self._hardlink_atomic(src_path, link_path,
+					"%s -> %s" % (link_path, src_path),
+					self.config.options.symlinks):
+				success = False
+				break
+
+		if success:
+			self._success()
+			self.returncode = os.EX_OK
+		else:
+			self.config.log_failure("%s\t%s\t%s" %
+				(self.cpv, self.distfile, msg))
+			self.config.file_failures[self.distfile] = self.cpv
+			self.returncode = 1
+
+		self.wait()
+
 	def _unlink_file(self, file_path, dir_info):
 		try:
 			os.unlink(file_path)
@@ -595,7 +628,7 @@ class FetchTask(CompositeTask):
 		else:
 			return st1.st_dev == st2.st_dev
 
-	def _hardlink_atomic(self, src, dest, dir_info):
+	def _hardlink_atomic(self, src, dest, dir_info, symlink=False):
 
 		head, tail = os.path.split(dest)
 		hardlink_tmp = os.path.join(head, ".%s._mirrordist_hardlink_.%s" % \
@@ -603,7 +636,10 @@ class FetchTask(CompositeTask):
 
 		try:
 			try:
-				os.link(src, hardlink_tmp)
+				if symlink:
+					os.symlink(src, hardlink_tmp)
+				else:
+					os.link(src, hardlink_tmp)
 			except OSError as e:
 				if e.errno != errno.EXDEV:
 					msg = "hardlink %s from %s failed: %s" % \
diff --git a/lib/portage/_emirrordist/main.py b/lib/portage/_emirrordist/main.py
index b63837b2a..0ae45ab6f 100644
--- a/lib/portage/_emirrordist/main.py
+++ b/lib/portage/_emirrordist/main.py
@@ -187,6 +187,12 @@ common_options = (
 		"action"   : "append",
 		"metavar"  : "FILE"
 	},
+	{
+		"longopt"  : "--symlinks",
+		"help"     : "use symlinks rather than hardlinks for linking "
+			"distfiles between layouts",
+		"action"   : "store_true"
+	},
 )
 
 def parse_args(args):
diff --git a/lib/portage/package/ebuild/fetch.py b/lib/portage/package/ebuild/fetch.py
index 4458796fc..5d0bc7355 100644
--- a/lib/portage/package/ebuild/fetch.py
+++ b/lib/portage/package/ebuild/fetch.py
@@ -27,13 +27,14 @@ except ImportError:
 
 import portage
 portage.proxy.lazyimport.lazyimport(globals(),
+	'glob:glob',
 	'portage.package.ebuild.config:check_config_instance,config',
 	'portage.package.ebuild.doebuild:doebuild_environment,' + \
 		'_doebuild_spawn',
 	'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs',
 	'portage.util:atomic_ofstream',
 	'portage.util.configparser:SafeConfigParser,read_configs,' +
-		'NoOptionError,ConfigParserError',
+		'ConfigParserError',
 	'portage.util._urlopen:urlopen',
 )
 
@@ -267,6 +268,9 @@ class FlatLayout(object):
 	def get_path(self, filename):
 		return filename
 
+	def get_filenames(self, distdir):
+		return iter(os.listdir(distdir))
+
 	@staticmethod
 	def verify_args(args):
 		return len(args) == 1
@@ -287,6 +291,16 @@ class FilenameHashLayout(object):
 			fnhash = fnhash[c:]
 		return ret + filename
 
+	def get_filenames(self, distdir):
+		pattern = ''
+		for c in self.cutoffs:
+			assert c % 4 == 0
+			c = c // 4
+			pattern += c * '[0-9a-f]' + '/'
+		pattern += '*'
+		return (x.rsplit('/', 1)[1]
+				for x in glob(os.path.join(distdir, pattern)))
+
 	@staticmethod
 	def verify_args(args):
 		if len(args) != 3:
@@ -322,7 +336,7 @@ class MirrorLayoutConfig(object):
 		for i in itertools.count():
 			try:
 				vals.append(tuple(cp.get('structure', '%d' % i).split()))
-			except NoOptionError:
+			except ConfigParserError:
 				break
 		self.structure = tuple(vals)
 
@@ -351,6 +365,19 @@ class MirrorLayoutConfig(object):
 			# fallback
 			return FlatLayout()
 
+	def get_all_layouts(self):
+		ret = []
+		for val in self.structure:
+			if not self.validate_structure(val):
+				raise ValueError("Unsupported structure: {}".format(val))
+			if val[0] == 'flat':
+				ret.append(FlatLayout(*val[1:]))
+			elif val[0] == 'filename-hash':
+				ret.append(FilenameHashLayout(*val[1:]))
+		if not ret:
+			ret.append(FlatLayout())
+		return ret
+
 
 def get_mirror_url(mirror_url, filename, cache_path=None):
 	"""